2024-12-09 06:18:58,582 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-09 06:18:58,597 main DEBUG Took 0.013144 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 06:18:58,598 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 06:18:58,598 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 06:18:58,599 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 06:18:58,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,607 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 06:18:58,621 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,623 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,624 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,625 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,625 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,627 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,628 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,628 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,629 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,630 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,630 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,631 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:18:58,632 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,632 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 06:18:58,633 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:18:58,635 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 06:18:58,636 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 06:18:58,637 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 06:18:58,638 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 06:18:58,638 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 06:18:58,646 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 06:18:58,649 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 06:18:58,652 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 06:18:58,653 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 06:18:58,654 main DEBUG createAppenders(={Console}) 2024-12-09 06:18:58,655 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-12-09 06:18:58,655 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-09 06:18:58,656 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-12-09 06:18:58,657 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 06:18:58,657 main DEBUG OutputStream closed 2024-12-09 06:18:58,657 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 06:18:58,658 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 06:18:58,658 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-12-09 06:18:58,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 06:18:58,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 06:18:58,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 06:18:58,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 06:18:58,740 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 06:18:58,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 06:18:58,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 06:18:58,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 06:18:58,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 06:18:58,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 06:18:58,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 06:18:58,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 06:18:58,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 06:18:58,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 06:18:58,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 06:18:58,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 06:18:58,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 06:18:58,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 06:18:58,749 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 06:18:58,749 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-12-09 06:18:58,749 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 06:18:58,750 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-12-09T06:18:58,761 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-12-09 06:18:58,764 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 06:18:58,764 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T06:18:58,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=543, ProcessCount=11, AvailableMemoryMB=6499 2024-12-09T06:18:59,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185 2024-12-09T06:18:59,631 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T06:18:59,729 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T06:18:59,781 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-12-09T06:18:59,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725139826 2024-12-09T06:18:59,895 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725139826 2024-12-09T06:18:59,962 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:18:59,970 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-12-09T06:18:59,971 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-12-09T06:19:00,012 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => aea0f23f7b5f4c66013ed1db76b82ad1, NAME => 'testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185 2024-12-09T06:19:00,061 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T06:19:00,066 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:19:00,165 INFO [StoreOpener-aea0f23f7b5f4c66013ed1db76b82ad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,208 INFO [StoreOpener-aea0f23f7b5f4c66013ed1db76b82ad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea0f23f7b5f4c66013ed1db76b82ad1 columnFamilyName MyCF 2024-12-09T06:19:00,215 DEBUG [StoreOpener-aea0f23f7b5f4c66013ed1db76b82ad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:19:00,222 INFO [StoreOpener-aea0f23f7b5f4c66013ed1db76b82ad1-1 {}] regionserver.HStore(327): Store=aea0f23f7b5f4c66013ed1db76b82ad1/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:19:00,226 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/data/default/testLockupAroundBadAssignSync/aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/data/default/testLockupAroundBadAssignSync/aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,234 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,234 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,240 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aea0f23f7b5f4c66013ed1db76b82ad1 2024-12-09T06:19:00,249 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/data/default/testLockupAroundBadAssignSync/aea0f23f7b5f4c66013ed1db76b82ad1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:19:00,250 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aea0f23f7b5f4c66013ed1db76b82ad1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65857428, jitterRate=-0.018647849559783936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T06:19:00,256 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T06:19:00,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aea0f23f7b5f4c66013ed1db76b82ad1: Writing region info on filesystem at 1733725140127Initializing all the Stores at 1733725140129 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1733725140130 (+1 ms)Cleaning up temporary data from old regions at 1733725140235 (+105 ms)Region opened successfully at 1733725140265 (+30 ms) 2024-12-09T06:19:00,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725139826) roll requested 2024-12-09T06:19:00,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140315 2024-12-09T06:19:00,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725139826 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140315 2024-12-09T06:19:00,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140315) roll requested 2024-12-09T06:19:00,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140328 2024-12-09T06:19:00,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140328 2024-12-09T06:19:00,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140328) roll requested 2024-12-09T06:19:00,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140341 2024-12-09T06:19:00,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140341 2024-12-09T06:19:00,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140341) roll requested 2024-12-09T06:19:00,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140356 2024-12-09T06:19:00,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140356 2024-12-09T06:19:00,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140356) roll requested 2024-12-09T06:19:00,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140368 2024-12-09T06:19:00,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140368 2024-12-09T06:19:00,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140368) roll requested 2024-12-09T06:19:00,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140386 2024-12-09T06:19:00,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140386 2024-12-09T06:19:00,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140386) roll requested 2024-12-09T06:19:00,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140406 2024-12-09T06:19:00,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140406 2024-12-09T06:19:00,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140406) roll requested 2024-12-09T06:19:00,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140418 2024-12-09T06:19:00,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140418 2024-12-09T06:19:00,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140418) roll requested 2024-12-09T06:19:00,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140430 2024-12-09T06:19:00,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140430 2024-12-09T06:19:00,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140430) roll requested 2024-12-09T06:19:00,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140442 2024-12-09T06:19:00,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140442 2024-12-09T06:19:00,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140442) roll requested 2024-12-09T06:19:00,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140454 2024-12-09T06:19:00,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140454 2024-12-09T06:19:00,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140454) roll requested 2024-12-09T06:19:00,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140463 2024-12-09T06:19:00,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140463 2024-12-09T06:19:00,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140463) roll requested 2024-12-09T06:19:00,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140473 2024-12-09T06:19:00,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140473 2024-12-09T06:19:00,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140473) roll requested 2024-12-09T06:19:00,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140484 2024-12-09T06:19:00,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140484 2024-12-09T06:19:00,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140484) roll requested 2024-12-09T06:19:00,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140494 2024-12-09T06:19:00,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140494 2024-12-09T06:19:00,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140494) roll requested 2024-12-09T06:19:00,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140510 2024-12-09T06:19:00,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140510 2024-12-09T06:19:00,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140510) roll requested 2024-12-09T06:19:00,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140519 2024-12-09T06:19:00,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140519 2024-12-09T06:19:00,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140519) roll requested 2024-12-09T06:19:00,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140529 2024-12-09T06:19:00,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140529 2024-12-09T06:19:00,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140529) roll requested 2024-12-09T06:19:00,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140537 2024-12-09T06:19:00,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140537 2024-12-09T06:19:00,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140537) roll requested 2024-12-09T06:19:00,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140548 2024-12-09T06:19:00,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140548 2024-12-09T06:19:00,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140548) roll requested 2024-12-09T06:19:00,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140559 2024-12-09T06:19:00,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140559 2024-12-09T06:19:00,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140559) roll requested 2024-12-09T06:19:00,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140574 2024-12-09T06:19:00,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140574 2024-12-09T06:19:00,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140574) roll requested 2024-12-09T06:19:00,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140587 2024-12-09T06:19:00,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140587 2024-12-09T06:19:00,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140587) roll requested 2024-12-09T06:19:00,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140594 2024-12-09T06:19:00,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140594 2024-12-09T06:19:00,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140594) roll requested 2024-12-09T06:19:00,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140602 2024-12-09T06:19:00,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140602 2024-12-09T06:19:00,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140602) roll requested 2024-12-09T06:19:00,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140610 2024-12-09T06:19:00,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140610 2024-12-09T06:19:00,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140610) roll requested 2024-12-09T06:19:00,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140618 2024-12-09T06:19:00,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140618 2024-12-09T06:19:00,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140618) roll requested 2024-12-09T06:19:00,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140626 2024-12-09T06:19:00,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140626 2024-12-09T06:19:00,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140626) roll requested 2024-12-09T06:19:00,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140634 2024-12-09T06:19:00,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140634 2024-12-09T06:19:00,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140634) roll requested 2024-12-09T06:19:00,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140645 2024-12-09T06:19:00,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140645 2024-12-09T06:19:00,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140645) roll requested 2024-12-09T06:19:00,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140653 2024-12-09T06:19:00,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140653 2024-12-09T06:19:00,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140653) roll requested 2024-12-09T06:19:00,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140664 2024-12-09T06:19:00,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140664 2024-12-09T06:19:00,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140664) roll requested 2024-12-09T06:19:00,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140672 2024-12-09T06:19:00,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140672 2024-12-09T06:19:00,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140672) roll requested 2024-12-09T06:19:00,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140681 2024-12-09T06:19:00,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140681 2024-12-09T06:19:00,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140681) roll requested 2024-12-09T06:19:00,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140689 2024-12-09T06:19:00,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140689 2024-12-09T06:19:00,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140689) roll requested 2024-12-09T06:19:00,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140697 2024-12-09T06:19:00,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140697 2024-12-09T06:19:00,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140697) roll requested 2024-12-09T06:19:00,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140706 2024-12-09T06:19:00,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140706 2024-12-09T06:19:00,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140706) roll requested 2024-12-09T06:19:00,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140719 2024-12-09T06:19:00,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140719 2024-12-09T06:19:00,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140719) roll requested 2024-12-09T06:19:00,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140727 2024-12-09T06:19:00,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140727 2024-12-09T06:19:00,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140727) roll requested 2024-12-09T06:19:00,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140739 2024-12-09T06:19:00,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140739 2024-12-09T06:19:00,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140739) roll requested 2024-12-09T06:19:00,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140746 2024-12-09T06:19:00,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140746 2024-12-09T06:19:00,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140746) roll requested 2024-12-09T06:19:00,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140754 2024-12-09T06:19:00,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140754 2024-12-09T06:19:00,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140754) roll requested 2024-12-09T06:19:00,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140764 2024-12-09T06:19:00,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140764 2024-12-09T06:19:00,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140764) roll requested 2024-12-09T06:19:00,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140773 2024-12-09T06:19:00,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140773 2024-12-09T06:19:00,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140773) roll requested 2024-12-09T06:19:00,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140783 2024-12-09T06:19:00,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140783 2024-12-09T06:19:00,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140783) roll requested 2024-12-09T06:19:00,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140789 2024-12-09T06:19:00,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140789 2024-12-09T06:19:00,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140789) roll requested 2024-12-09T06:19:00,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140797 2024-12-09T06:19:00,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140797 2024-12-09T06:19:00,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140797) roll requested 2024-12-09T06:19:00,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140805 2024-12-09T06:19:00,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140805 2024-12-09T06:19:00,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140805) roll requested 2024-12-09T06:19:00,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140814 2024-12-09T06:19:00,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140814 2024-12-09T06:19:00,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140814) roll requested 2024-12-09T06:19:00,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140824 2024-12-09T06:19:00,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140824 2024-12-09T06:19:00,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140824) roll requested 2024-12-09T06:19:00,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140831 2024-12-09T06:19:00,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140831 2024-12-09T06:19:00,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140831) roll requested 2024-12-09T06:19:00,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140840 2024-12-09T06:19:00,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140840 2024-12-09T06:19:00,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140840) roll requested 2024-12-09T06:19:00,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140847 2024-12-09T06:19:00,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140847 2024-12-09T06:19:00,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140847) roll requested 2024-12-09T06:19:00,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140853 2024-12-09T06:19:00,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140853 2024-12-09T06:19:00,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140853) roll requested 2024-12-09T06:19:00,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140862 2024-12-09T06:19:00,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140862 2024-12-09T06:19:00,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140862) roll requested 2024-12-09T06:19:00,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140871 2024-12-09T06:19:00,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140871 2024-12-09T06:19:00,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140871) roll requested 2024-12-09T06:19:00,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140881 2024-12-09T06:19:00,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140881 2024-12-09T06:19:00,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140881) roll requested 2024-12-09T06:19:00,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140888 2024-12-09T06:19:00,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140888 2024-12-09T06:19:00,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140888) roll requested 2024-12-09T06:19:00,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140895 2024-12-09T06:19:00,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140895 2024-12-09T06:19:00,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140895) roll requested 2024-12-09T06:19:00,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140902 2024-12-09T06:19:00,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140902 2024-12-09T06:19:00,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140902) roll requested 2024-12-09T06:19:00,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140910 2024-12-09T06:19:00,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140910 2024-12-09T06:19:00,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140910) roll requested 2024-12-09T06:19:00,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140921 2024-12-09T06:19:00,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140921 2024-12-09T06:19:00,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140921) roll requested 2024-12-09T06:19:00,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140933 2024-12-09T06:19:00,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140933 2024-12-09T06:19:00,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140933) roll requested 2024-12-09T06:19:00,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140950 2024-12-09T06:19:00,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140950 2024-12-09T06:19:00,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140950) roll requested 2024-12-09T06:19:00,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140958 2024-12-09T06:19:00,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140958 2024-12-09T06:19:00,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140958) roll requested 2024-12-09T06:19:00,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140969 2024-12-09T06:19:00,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140969 2024-12-09T06:19:00,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140969) roll requested 2024-12-09T06:19:00,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140981 2024-12-09T06:19:00,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140981 2024-12-09T06:19:00,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140981) roll requested 2024-12-09T06:19:00,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140987 2024-12-09T06:19:00,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:00,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140987 2024-12-09T06:19:00,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:00,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:00,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140987) roll requested 2024-12-09T06:19:00,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725140993 2024-12-09T06:19:01,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140993 2024-12-09T06:19:01,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725140993) roll requested 2024-12-09T06:19:01,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141004 2024-12-09T06:19:01,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725140993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141004 2024-12-09T06:19:01,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141004) roll requested 2024-12-09T06:19:01,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141010 2024-12-09T06:19:01,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141010 2024-12-09T06:19:01,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141010) roll requested 2024-12-09T06:19:01,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141016 2024-12-09T06:19:01,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141016 2024-12-09T06:19:01,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141016) roll requested 2024-12-09T06:19:01,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141024 2024-12-09T06:19:01,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141024 2024-12-09T06:19:01,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141024) roll requested 2024-12-09T06:19:01,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141032 2024-12-09T06:19:01,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141032 2024-12-09T06:19:01,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141032) roll requested 2024-12-09T06:19:01,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141046 2024-12-09T06:19:01,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141046 2024-12-09T06:19:01,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141046) roll requested 2024-12-09T06:19:01,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141056 2024-12-09T06:19:01,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141056 2024-12-09T06:19:01,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141056) roll requested 2024-12-09T06:19:01,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141064 2024-12-09T06:19:01,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141064 2024-12-09T06:19:01,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141064) roll requested 2024-12-09T06:19:01,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141071 2024-12-09T06:19:01,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141071 2024-12-09T06:19:01,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141071) roll requested 2024-12-09T06:19:01,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141077 2024-12-09T06:19:01,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141077 2024-12-09T06:19:01,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141077) roll requested 2024-12-09T06:19:01,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141084 2024-12-09T06:19:01,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141084 2024-12-09T06:19:01,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141084) roll requested 2024-12-09T06:19:01,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141095 2024-12-09T06:19:01,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141095 2024-12-09T06:19:01,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141095) roll requested 2024-12-09T06:19:01,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141103 2024-12-09T06:19:01,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141103 2024-12-09T06:19:01,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141103) roll requested 2024-12-09T06:19:01,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141111 2024-12-09T06:19:01,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141111 2024-12-09T06:19:01,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141111) roll requested 2024-12-09T06:19:01,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141121 2024-12-09T06:19:01,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141121 2024-12-09T06:19:01,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141121) roll requested 2024-12-09T06:19:01,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141129 2024-12-09T06:19:01,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141129 2024-12-09T06:19:01,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141129) roll requested 2024-12-09T06:19:01,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141137 2024-12-09T06:19:01,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141137 2024-12-09T06:19:01,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141137) roll requested 2024-12-09T06:19:01,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141145 2024-12-09T06:19:01,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141145 2024-12-09T06:19:01,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141145) roll requested 2024-12-09T06:19:01,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141155 2024-12-09T06:19:01,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141155 2024-12-09T06:19:01,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141155) roll requested 2024-12-09T06:19:01,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141162 2024-12-09T06:19:01,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141162 2024-12-09T06:19:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141162) roll requested 2024-12-09T06:19:01,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141172 2024-12-09T06:19:01,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141172 2024-12-09T06:19:01,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141162 is not closed yet, will try archiving it next time 2024-12-09T06:19:01,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141172) roll requested 2024-12-09T06:19:01,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141182 2024-12-09T06:19:01,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141182 2024-12-09T06:19:01,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141182) roll requested 2024-12-09T06:19:01,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141190 2024-12-09T06:19:01,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141190 2024-12-09T06:19:01,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141190) roll requested 2024-12-09T06:19:01,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141199 2024-12-09T06:19:01,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141199 2024-12-09T06:19:01,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141199) roll requested 2024-12-09T06:19:01,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141208 2024-12-09T06:19:01,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141208 2024-12-09T06:19:01,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141208) roll requested 2024-12-09T06:19:01,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141219 2024-12-09T06:19:01,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141219 2024-12-09T06:19:01,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141219) roll requested 2024-12-09T06:19:01,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141226 2024-12-09T06:19:01,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141226 2024-12-09T06:19:01,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141226) roll requested 2024-12-09T06:19:01,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141236 2024-12-09T06:19:01,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141236 2024-12-09T06:19:01,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141236) roll requested 2024-12-09T06:19:01,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141244 2024-12-09T06:19:01,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141244 2024-12-09T06:19:01,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141244) roll requested 2024-12-09T06:19:01,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141251 2024-12-09T06:19:01,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141251 2024-12-09T06:19:01,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141251) roll requested 2024-12-09T06:19:01,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141262 2024-12-09T06:19:01,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141262 2024-12-09T06:19:01,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141262) roll requested 2024-12-09T06:19:01,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141274 2024-12-09T06:19:01,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141274 2024-12-09T06:19:01,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141274) roll requested 2024-12-09T06:19:01,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141284 2024-12-09T06:19:01,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141284 2024-12-09T06:19:01,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141284) roll requested 2024-12-09T06:19:01,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141292 2024-12-09T06:19:01,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141292 2024-12-09T06:19:01,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141292) roll requested 2024-12-09T06:19:01,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141301 2024-12-09T06:19:01,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141301 2024-12-09T06:19:01,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141301) roll requested 2024-12-09T06:19:01,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141310 2024-12-09T06:19:01,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141310 2024-12-09T06:19:01,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141310) roll requested 2024-12-09T06:19:01,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141319 2024-12-09T06:19:01,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141319 2024-12-09T06:19:01,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141319) roll requested 2024-12-09T06:19:01,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141329 2024-12-09T06:19:01,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141329 2024-12-09T06:19:01,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141329) roll requested 2024-12-09T06:19:01,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141338 2024-12-09T06:19:01,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141338 2024-12-09T06:19:01,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141338) roll requested 2024-12-09T06:19:01,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141349 2024-12-09T06:19:01,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141349 2024-12-09T06:19:01,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141349) roll requested 2024-12-09T06:19:01,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141363 2024-12-09T06:19:01,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141363 2024-12-09T06:19:01,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141363) roll requested 2024-12-09T06:19:01,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141372 2024-12-09T06:19:01,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141372 2024-12-09T06:19:01,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141372) roll requested 2024-12-09T06:19:01,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141381 2024-12-09T06:19:01,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141381 2024-12-09T06:19:01,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141381) roll requested 2024-12-09T06:19:01,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141395 2024-12-09T06:19:01,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141395 2024-12-09T06:19:01,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141395) roll requested 2024-12-09T06:19:01,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141405 2024-12-09T06:19:01,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141405 2024-12-09T06:19:01,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141405) roll requested 2024-12-09T06:19:01,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141415 2024-12-09T06:19:01,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141415 2024-12-09T06:19:01,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141415) roll requested 2024-12-09T06:19:01,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141421 2024-12-09T06:19:01,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141421 2024-12-09T06:19:01,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141421) roll requested 2024-12-09T06:19:01,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141428 2024-12-09T06:19:01,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141428 2024-12-09T06:19:01,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141428) roll requested 2024-12-09T06:19:01,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141434 2024-12-09T06:19:01,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141434 2024-12-09T06:19:01,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141434) roll requested 2024-12-09T06:19:01,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141442 2024-12-09T06:19:01,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141442 2024-12-09T06:19:01,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141442) roll requested 2024-12-09T06:19:01,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141449 2024-12-09T06:19:01,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141449 2024-12-09T06:19:01,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141449) roll requested 2024-12-09T06:19:01,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141455 2024-12-09T06:19:01,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141455 2024-12-09T06:19:01,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141455) roll requested 2024-12-09T06:19:01,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141462 2024-12-09T06:19:01,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141462 2024-12-09T06:19:01,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141462) roll requested 2024-12-09T06:19:01,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141469 2024-12-09T06:19:01,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141469 2024-12-09T06:19:01,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141469) roll requested 2024-12-09T06:19:01,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141476 2024-12-09T06:19:01,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141476 2024-12-09T06:19:01,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141476) roll requested 2024-12-09T06:19:01,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141483 2024-12-09T06:19:01,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141483 2024-12-09T06:19:01,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141483) roll requested 2024-12-09T06:19:01,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141491 2024-12-09T06:19:01,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141491 2024-12-09T06:19:01,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141491) roll requested 2024-12-09T06:19:01,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141502 2024-12-09T06:19:01,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141502 2024-12-09T06:19:01,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141502) roll requested 2024-12-09T06:19:01,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141509 2024-12-09T06:19:01,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141509 2024-12-09T06:19:01,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141509) roll requested 2024-12-09T06:19:01,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141515 2024-12-09T06:19:01,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141515 2024-12-09T06:19:01,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141515) roll requested 2024-12-09T06:19:01,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141523 2024-12-09T06:19:01,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141523 2024-12-09T06:19:01,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141523) roll requested 2024-12-09T06:19:01,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141535 2024-12-09T06:19:01,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141535 2024-12-09T06:19:01,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141535) roll requested 2024-12-09T06:19:01,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141542 2024-12-09T06:19:01,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141542 2024-12-09T06:19:01,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141542) roll requested 2024-12-09T06:19:01,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141695 2024-12-09T06:19:01,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141695 2024-12-09T06:19:01,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141695) roll requested 2024-12-09T06:19:01,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141701 2024-12-09T06:19:01,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141701 2024-12-09T06:19:01,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141701) roll requested 2024-12-09T06:19:01,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141706 2024-12-09T06:19:01,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141706 2024-12-09T06:19:01,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141706) roll requested 2024-12-09T06:19:01,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141712 2024-12-09T06:19:01,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141712 2024-12-09T06:19:01,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141712) roll requested 2024-12-09T06:19:01,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141719 2024-12-09T06:19:01,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141719 2024-12-09T06:19:01,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141719) roll requested 2024-12-09T06:19:01,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141727 2024-12-09T06:19:01,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141727 2024-12-09T06:19:01,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141727) roll requested 2024-12-09T06:19:01,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141733 2024-12-09T06:19:01,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141733 2024-12-09T06:19:01,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141733) roll requested 2024-12-09T06:19:01,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141739 2024-12-09T06:19:01,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141739 2024-12-09T06:19:01,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141739) roll requested 2024-12-09T06:19:01,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141747 2024-12-09T06:19:01,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141747 2024-12-09T06:19:01,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141747) roll requested 2024-12-09T06:19:01,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141756 2024-12-09T06:19:01,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141756 2024-12-09T06:19:01,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141756) roll requested 2024-12-09T06:19:01,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141767 2024-12-09T06:19:01,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141767 2024-12-09T06:19:01,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141767) roll requested 2024-12-09T06:19:01,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141776 2024-12-09T06:19:01,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141776 2024-12-09T06:19:01,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141776) roll requested 2024-12-09T06:19:01,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141788 2024-12-09T06:19:01,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141788 2024-12-09T06:19:01,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141788) roll requested 2024-12-09T06:19:01,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141797 2024-12-09T06:19:01,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141797 2024-12-09T06:19:01,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141797) roll requested 2024-12-09T06:19:01,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141808 2024-12-09T06:19:01,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141808 2024-12-09T06:19:01,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141808) roll requested 2024-12-09T06:19:01,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141816 2024-12-09T06:19:01,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141816 2024-12-09T06:19:01,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141816) roll requested 2024-12-09T06:19:01,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141825 2024-12-09T06:19:01,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141825 2024-12-09T06:19:01,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141825) roll requested 2024-12-09T06:19:01,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141833 2024-12-09T06:19:01,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141833 2024-12-09T06:19:01,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141833) roll requested 2024-12-09T06:19:01,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141841 2024-12-09T06:19:01,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141841 2024-12-09T06:19:01,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141841) roll requested 2024-12-09T06:19:01,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141851 2024-12-09T06:19:01,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141851 2024-12-09T06:19:01,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141851) roll requested 2024-12-09T06:19:01,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141860 2024-12-09T06:19:01,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141860 2024-12-09T06:19:01,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141860) roll requested 2024-12-09T06:19:01,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141876 2024-12-09T06:19:01,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141876 2024-12-09T06:19:01,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141876) roll requested 2024-12-09T06:19:01,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141884 2024-12-09T06:19:01,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141884 2024-12-09T06:19:01,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141884) roll requested 2024-12-09T06:19:01,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141896 2024-12-09T06:19:01,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141896 2024-12-09T06:19:01,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141896) roll requested 2024-12-09T06:19:01,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141902 2024-12-09T06:19:01,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141902 2024-12-09T06:19:01,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141902) roll requested 2024-12-09T06:19:01,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141909 2024-12-09T06:19:01,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141909 2024-12-09T06:19:01,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141909) roll requested 2024-12-09T06:19:01,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141916 2024-12-09T06:19:01,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141916 2024-12-09T06:19:01,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141916) roll requested 2024-12-09T06:19:01,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141925 2024-12-09T06:19:01,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141925 2024-12-09T06:19:01,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141925) roll requested 2024-12-09T06:19:01,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141931 2024-12-09T06:19:01,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141931 2024-12-09T06:19:01,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141931) roll requested 2024-12-09T06:19:01,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141938 2024-12-09T06:19:01,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141938 2024-12-09T06:19:01,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141938) roll requested 2024-12-09T06:19:01,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141945 2024-12-09T06:19:01,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141945 2024-12-09T06:19:01,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141945) roll requested 2024-12-09T06:19:01,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141951 2024-12-09T06:19:01,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141951 2024-12-09T06:19:01,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141951) roll requested 2024-12-09T06:19:01,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141957 2024-12-09T06:19:01,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141957 2024-12-09T06:19:01,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141957) roll requested 2024-12-09T06:19:01,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141963 2024-12-09T06:19:01,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141963 2024-12-09T06:19:01,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141963) roll requested 2024-12-09T06:19:01,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141970 2024-12-09T06:19:01,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141970 2024-12-09T06:19:01,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141970) roll requested 2024-12-09T06:19:01,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141976 2024-12-09T06:19:01,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141976 2024-12-09T06:19:01,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141976) roll requested 2024-12-09T06:19:01,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141982 2024-12-09T06:19:01,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141982 2024-12-09T06:19:01,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141982) roll requested 2024-12-09T06:19:01,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141992 2024-12-09T06:19:01,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:01,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141992 2024-12-09T06:19:01,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:01,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:01,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141992) roll requested 2024-12-09T06:19:01,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725141999 2024-12-09T06:19:02,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141999 2024-12-09T06:19:02,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725141999) roll requested 2024-12-09T06:19:02,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142010 2024-12-09T06:19:02,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725141999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142010 2024-12-09T06:19:02,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142010) roll requested 2024-12-09T06:19:02,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142017 2024-12-09T06:19:02,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142017 2024-12-09T06:19:02,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142017) roll requested 2024-12-09T06:19:02,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142025 2024-12-09T06:19:02,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142025 2024-12-09T06:19:02,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142025) roll requested 2024-12-09T06:19:02,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142032 2024-12-09T06:19:02,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142032 2024-12-09T06:19:02,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142032) roll requested 2024-12-09T06:19:02,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142041 2024-12-09T06:19:02,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142041 2024-12-09T06:19:02,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142041) roll requested 2024-12-09T06:19:02,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142048 2024-12-09T06:19:02,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142048 2024-12-09T06:19:02,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142048) roll requested 2024-12-09T06:19:02,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142056 2024-12-09T06:19:02,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142056 2024-12-09T06:19:02,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142056) roll requested 2024-12-09T06:19:02,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142064 2024-12-09T06:19:02,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142064 2024-12-09T06:19:02,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142064) roll requested 2024-12-09T06:19:02,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142071 2024-12-09T06:19:02,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142071 2024-12-09T06:19:02,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142071) roll requested 2024-12-09T06:19:02,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142081 2024-12-09T06:19:02,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142081 2024-12-09T06:19:02,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142081) roll requested 2024-12-09T06:19:02,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142088 2024-12-09T06:19:02,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142088 2024-12-09T06:19:02,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142088) roll requested 2024-12-09T06:19:02,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142096 2024-12-09T06:19:02,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142096 2024-12-09T06:19:02,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142096) roll requested 2024-12-09T06:19:02,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142102 2024-12-09T06:19:02,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142102 2024-12-09T06:19:02,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142102) roll requested 2024-12-09T06:19:02,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142110 2024-12-09T06:19:02,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142110 2024-12-09T06:19:02,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142110) roll requested 2024-12-09T06:19:02,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142122 2024-12-09T06:19:02,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142122 2024-12-09T06:19:02,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142122) roll requested 2024-12-09T06:19:02,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142129 2024-12-09T06:19:02,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142129 2024-12-09T06:19:02,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142129) roll requested 2024-12-09T06:19:02,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142136 2024-12-09T06:19:02,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142136 2024-12-09T06:19:02,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142136) roll requested 2024-12-09T06:19:02,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142142 2024-12-09T06:19:02,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142142 2024-12-09T06:19:02,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142142) roll requested 2024-12-09T06:19:02,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142154 2024-12-09T06:19:02,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142154 2024-12-09T06:19:02,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142154) roll requested 2024-12-09T06:19:02,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142160 2024-12-09T06:19:02,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142160 2024-12-09T06:19:02,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142160) roll requested 2024-12-09T06:19:02,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142168 2024-12-09T06:19:02,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142168 2024-12-09T06:19:02,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142168) roll requested 2024-12-09T06:19:02,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142177 2024-12-09T06:19:02,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142177 2024-12-09T06:19:02,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142177) roll requested 2024-12-09T06:19:02,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142185 2024-12-09T06:19:02,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142185 2024-12-09T06:19:02,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142185) roll requested 2024-12-09T06:19:02,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142201 2024-12-09T06:19:02,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142201 2024-12-09T06:19:02,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142201) roll requested 2024-12-09T06:19:02,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142213 2024-12-09T06:19:02,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142213 2024-12-09T06:19:02,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142213) roll requested 2024-12-09T06:19:02,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142219 2024-12-09T06:19:02,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142219 2024-12-09T06:19:02,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142219) roll requested 2024-12-09T06:19:02,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142229 2024-12-09T06:19:02,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142229 2024-12-09T06:19:02,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142229) roll requested 2024-12-09T06:19:02,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142238 2024-12-09T06:19:02,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142238 2024-12-09T06:19:02,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142238) roll requested 2024-12-09T06:19:02,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142246 2024-12-09T06:19:02,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142246 2024-12-09T06:19:02,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142246) roll requested 2024-12-09T06:19:02,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142255 2024-12-09T06:19:02,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142255 2024-12-09T06:19:02,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142255) roll requested 2024-12-09T06:19:02,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142262 2024-12-09T06:19:02,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142262 2024-12-09T06:19:02,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142262) roll requested 2024-12-09T06:19:02,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142273 2024-12-09T06:19:02,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142273 2024-12-09T06:19:02,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142273) roll requested 2024-12-09T06:19:02,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142280 2024-12-09T06:19:02,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142280 2024-12-09T06:19:02,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142280) roll requested 2024-12-09T06:19:02,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142288 2024-12-09T06:19:02,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142288 2024-12-09T06:19:02,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142288) roll requested 2024-12-09T06:19:02,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142295 2024-12-09T06:19:02,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142295 2024-12-09T06:19:02,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142295) roll requested 2024-12-09T06:19:02,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142302 2024-12-09T06:19:02,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142302 2024-12-09T06:19:02,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142302) roll requested 2024-12-09T06:19:02,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142309 2024-12-09T06:19:02,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142309 2024-12-09T06:19:02,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142309) roll requested 2024-12-09T06:19:02,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142317 2024-12-09T06:19:02,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142317 2024-12-09T06:19:02,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142317) roll requested 2024-12-09T06:19:02,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142325 2024-12-09T06:19:02,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142325 2024-12-09T06:19:02,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142325) roll requested 2024-12-09T06:19:02,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142333 2024-12-09T06:19:02,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142333 2024-12-09T06:19:02,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142333) roll requested 2024-12-09T06:19:02,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142341 2024-12-09T06:19:02,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142341 2024-12-09T06:19:02,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142341) roll requested 2024-12-09T06:19:02,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142352 2024-12-09T06:19:02,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142352 2024-12-09T06:19:02,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142352) roll requested 2024-12-09T06:19:02,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142359 2024-12-09T06:19:02,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142359 2024-12-09T06:19:02,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142359) roll requested 2024-12-09T06:19:02,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142366 2024-12-09T06:19:02,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142366 2024-12-09T06:19:02,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142366) roll requested 2024-12-09T06:19:02,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142373 2024-12-09T06:19:02,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142373 2024-12-09T06:19:02,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142373) roll requested 2024-12-09T06:19:02,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142379 2024-12-09T06:19:02,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142379 2024-12-09T06:19:02,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142379) roll requested 2024-12-09T06:19:02,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142392 2024-12-09T06:19:02,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142392 2024-12-09T06:19:02,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142392) roll requested 2024-12-09T06:19:02,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142399 2024-12-09T06:19:02,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142399 2024-12-09T06:19:02,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142399) roll requested 2024-12-09T06:19:02,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142406 2024-12-09T06:19:02,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142406 2024-12-09T06:19:02,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142406) roll requested 2024-12-09T06:19:02,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142412 2024-12-09T06:19:02,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142412 2024-12-09T06:19:02,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142412) roll requested 2024-12-09T06:19:02,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142421 2024-12-09T06:19:02,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142421 2024-12-09T06:19:02,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142421) roll requested 2024-12-09T06:19:02,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142429 2024-12-09T06:19:02,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142429 2024-12-09T06:19:02,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142429) roll requested 2024-12-09T06:19:02,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142436 2024-12-09T06:19:02,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142436 2024-12-09T06:19:02,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142436) roll requested 2024-12-09T06:19:02,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142443 2024-12-09T06:19:02,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142443 2024-12-09T06:19:02,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142443) roll requested 2024-12-09T06:19:02,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142448 2024-12-09T06:19:02,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142448 2024-12-09T06:19:02,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142448) roll requested 2024-12-09T06:19:02,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142464 2024-12-09T06:19:02,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142464 2024-12-09T06:19:02,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142464) roll requested 2024-12-09T06:19:02,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142473 2024-12-09T06:19:02,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142473 2024-12-09T06:19:02,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142473) roll requested 2024-12-09T06:19:02,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142479 2024-12-09T06:19:02,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142479 2024-12-09T06:19:02,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142473 is not closed yet, will try archiving it next time 2024-12-09T06:19:02,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142479) roll requested 2024-12-09T06:19:02,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142486 2024-12-09T06:19:02,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142486 2024-12-09T06:19:02,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142486) roll requested 2024-12-09T06:19:02,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142538 2024-12-09T06:19:02,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142538 2024-12-09T06:19:02,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142538) roll requested 2024-12-09T06:19:02,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142550 2024-12-09T06:19:02,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142550 2024-12-09T06:19:02,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142550) roll requested 2024-12-09T06:19:02,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142557 2024-12-09T06:19:02,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142557 2024-12-09T06:19:02,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142557) roll requested 2024-12-09T06:19:02,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142564 2024-12-09T06:19:02,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142564 2024-12-09T06:19:02,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142564) roll requested 2024-12-09T06:19:02,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142577 2024-12-09T06:19:02,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142577 2024-12-09T06:19:02,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142577) roll requested 2024-12-09T06:19:02,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142595 2024-12-09T06:19:02,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142595 2024-12-09T06:19:02,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142595) roll requested 2024-12-09T06:19:02,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142604 2024-12-09T06:19:02,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142604 2024-12-09T06:19:02,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142604) roll requested 2024-12-09T06:19:02,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142611 2024-12-09T06:19:02,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142611 2024-12-09T06:19:02,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142611) roll requested 2024-12-09T06:19:02,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142618 2024-12-09T06:19:02,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142618 2024-12-09T06:19:02,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142618) roll requested 2024-12-09T06:19:02,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142625 2024-12-09T06:19:02,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142625 2024-12-09T06:19:02,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142625) roll requested 2024-12-09T06:19:02,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142630 2024-12-09T06:19:02,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142630 2024-12-09T06:19:02,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142630) roll requested 2024-12-09T06:19:02,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142637 2024-12-09T06:19:02,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142637 2024-12-09T06:19:02,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142637) roll requested 2024-12-09T06:19:02,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142642 2024-12-09T06:19:02,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142642 2024-12-09T06:19:02,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142642) roll requested 2024-12-09T06:19:02,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142653 2024-12-09T06:19:02,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142653 2024-12-09T06:19:02,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142653) roll requested 2024-12-09T06:19:02,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142663 2024-12-09T06:19:02,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142663 2024-12-09T06:19:02,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142663) roll requested 2024-12-09T06:19:02,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142677 2024-12-09T06:19:02,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142677 2024-12-09T06:19:02,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142677) roll requested 2024-12-09T06:19:02,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142690 2024-12-09T06:19:02,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142690 2024-12-09T06:19:02,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142690) roll requested 2024-12-09T06:19:02,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142697 2024-12-09T06:19:02,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142697 2024-12-09T06:19:02,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142697) roll requested 2024-12-09T06:19:02,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142705 2024-12-09T06:19:02,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142705 2024-12-09T06:19:02,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142705) roll requested 2024-12-09T06:19:02,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142717 2024-12-09T06:19:02,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142717 2024-12-09T06:19:02,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142717) roll requested 2024-12-09T06:19:02,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142727 2024-12-09T06:19:02,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142727 2024-12-09T06:19:02,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142727) roll requested 2024-12-09T06:19:02,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142735 2024-12-09T06:19:02,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142735 2024-12-09T06:19:02,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142735) roll requested 2024-12-09T06:19:02,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142747 2024-12-09T06:19:02,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142747 2024-12-09T06:19:02,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142747) roll requested 2024-12-09T06:19:02,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142757 2024-12-09T06:19:02,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142757 2024-12-09T06:19:02,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142757) roll requested 2024-12-09T06:19:02,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142767 2024-12-09T06:19:02,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142767 2024-12-09T06:19:02,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142767) roll requested 2024-12-09T06:19:02,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142781 2024-12-09T06:19:02,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142781 2024-12-09T06:19:02,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142781) roll requested 2024-12-09T06:19:02,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142789 2024-12-09T06:19:02,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142789 2024-12-09T06:19:02,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142789) roll requested 2024-12-09T06:19:02,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142795 2024-12-09T06:19:02,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142795 2024-12-09T06:19:02,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142795) roll requested 2024-12-09T06:19:02,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142801 2024-12-09T06:19:02,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142801 2024-12-09T06:19:02,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142801) roll requested 2024-12-09T06:19:02,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142807 2024-12-09T06:19:02,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142807 2024-12-09T06:19:02,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142807) roll requested 2024-12-09T06:19:02,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142814 2024-12-09T06:19:02,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142814 2024-12-09T06:19:02,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142814) roll requested 2024-12-09T06:19:02,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142821 2024-12-09T06:19:02,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142821 2024-12-09T06:19:02,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142821) roll requested 2024-12-09T06:19:02,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142829 2024-12-09T06:19:02,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142829 2024-12-09T06:19:02,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142829) roll requested 2024-12-09T06:19:02,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142836 2024-12-09T06:19:02,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142836 2024-12-09T06:19:02,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142836) roll requested 2024-12-09T06:19:02,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142845 2024-12-09T06:19:02,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142845 2024-12-09T06:19:02,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142845) roll requested 2024-12-09T06:19:02,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142855 2024-12-09T06:19:02,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142855 2024-12-09T06:19:02,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142855) roll requested 2024-12-09T06:19:02,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142863 2024-12-09T06:19:02,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142863 2024-12-09T06:19:02,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142863) roll requested 2024-12-09T06:19:02,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142868 2024-12-09T06:19:02,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142868 2024-12-09T06:19:02,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142868) roll requested 2024-12-09T06:19:02,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142874 2024-12-09T06:19:02,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142874 2024-12-09T06:19:02,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142874) roll requested 2024-12-09T06:19:02,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142880 2024-12-09T06:19:02,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142880 2024-12-09T06:19:02,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142880) roll requested 2024-12-09T06:19:02,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142888 2024-12-09T06:19:02,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142888 2024-12-09T06:19:02,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142888) roll requested 2024-12-09T06:19:02,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142896 2024-12-09T06:19:02,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142896 2024-12-09T06:19:02,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142896) roll requested 2024-12-09T06:19:02,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142904 2024-12-09T06:19:02,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142904 2024-12-09T06:19:02,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142904) roll requested 2024-12-09T06:19:02,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142913 2024-12-09T06:19:02,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142913 2024-12-09T06:19:02,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142913) roll requested 2024-12-09T06:19:02,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142922 2024-12-09T06:19:02,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142922 2024-12-09T06:19:02,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142922) roll requested 2024-12-09T06:19:02,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142932 2024-12-09T06:19:02,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142932 2024-12-09T06:19:02,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142932) roll requested 2024-12-09T06:19:02,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142941 2024-12-09T06:19:02,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142941 2024-12-09T06:19:02,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142941) roll requested 2024-12-09T06:19:02,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142952 2024-12-09T06:19:02,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142952 2024-12-09T06:19:02,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142952) roll requested 2024-12-09T06:19:02,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142964 2024-12-09T06:19:02,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142964 2024-12-09T06:19:02,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142964) roll requested 2024-12-09T06:19:02,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142970 2024-12-09T06:19:02,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142970 2024-12-09T06:19:02,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142970) roll requested 2024-12-09T06:19:02,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142983 2024-12-09T06:19:02,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142983 2024-12-09T06:19:02,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142983) roll requested 2024-12-09T06:19:02,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142991 2024-12-09T06:19:02,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:02,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142991 2024-12-09T06:19:02,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:02,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:02,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142991) roll requested 2024-12-09T06:19:02,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725142998 2024-12-09T06:19:03,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142998 2024-12-09T06:19:03,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725142998) roll requested 2024-12-09T06:19:03,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143005 2024-12-09T06:19:03,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725142998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143005 2024-12-09T06:19:03,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143005) roll requested 2024-12-09T06:19:03,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143020 2024-12-09T06:19:03,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143020 2024-12-09T06:19:03,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143020) roll requested 2024-12-09T06:19:03,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143039 2024-12-09T06:19:03,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143039 2024-12-09T06:19:03,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143039) roll requested 2024-12-09T06:19:03,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143049 2024-12-09T06:19:03,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143049 2024-12-09T06:19:03,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143049) roll requested 2024-12-09T06:19:03,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143060 2024-12-09T06:19:03,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143060 2024-12-09T06:19:03,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143060) roll requested 2024-12-09T06:19:03,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143067 2024-12-09T06:19:03,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143067 2024-12-09T06:19:03,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143067) roll requested 2024-12-09T06:19:03,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143072 2024-12-09T06:19:03,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143072 2024-12-09T06:19:03,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143072) roll requested 2024-12-09T06:19:03,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143077 2024-12-09T06:19:03,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143077 2024-12-09T06:19:03,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143077) roll requested 2024-12-09T06:19:03,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143084 2024-12-09T06:19:03,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143084 2024-12-09T06:19:03,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143084) roll requested 2024-12-09T06:19:03,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143091 2024-12-09T06:19:03,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143091 2024-12-09T06:19:03,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143091) roll requested 2024-12-09T06:19:03,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143098 2024-12-09T06:19:03,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143098 2024-12-09T06:19:03,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143098) roll requested 2024-12-09T06:19:03,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143105 2024-12-09T06:19:03,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143105 2024-12-09T06:19:03,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143105) roll requested 2024-12-09T06:19:03,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143113 2024-12-09T06:19:03,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143113 2024-12-09T06:19:03,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143113) roll requested 2024-12-09T06:19:03,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143129 2024-12-09T06:19:03,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143129 2024-12-09T06:19:03,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143129) roll requested 2024-12-09T06:19:03,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143143 2024-12-09T06:19:03,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143143 2024-12-09T06:19:03,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143143) roll requested 2024-12-09T06:19:03,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143149 2024-12-09T06:19:03,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143149 2024-12-09T06:19:03,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143149) roll requested 2024-12-09T06:19:03,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143155 2024-12-09T06:19:03,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143155 2024-12-09T06:19:03,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143155) roll requested 2024-12-09T06:19:03,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143169 2024-12-09T06:19:03,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143169 2024-12-09T06:19:03,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143169) roll requested 2024-12-09T06:19:03,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143175 2024-12-09T06:19:03,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143175 2024-12-09T06:19:03,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143175) roll requested 2024-12-09T06:19:03,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143185 2024-12-09T06:19:03,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143185 2024-12-09T06:19:03,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143185) roll requested 2024-12-09T06:19:03,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143193 2024-12-09T06:19:03,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143193 2024-12-09T06:19:03,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143193) roll requested 2024-12-09T06:19:03,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143201 2024-12-09T06:19:03,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143201 2024-12-09T06:19:03,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143201) roll requested 2024-12-09T06:19:03,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143209 2024-12-09T06:19:03,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143209 2024-12-09T06:19:03,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143209) roll requested 2024-12-09T06:19:03,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143215 2024-12-09T06:19:03,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143215 2024-12-09T06:19:03,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143215) roll requested 2024-12-09T06:19:03,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143221 2024-12-09T06:19:03,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143221 2024-12-09T06:19:03,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143221) roll requested 2024-12-09T06:19:03,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143226 2024-12-09T06:19:03,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143226 2024-12-09T06:19:03,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143226) roll requested 2024-12-09T06:19:03,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143232 2024-12-09T06:19:03,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143232 2024-12-09T06:19:03,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143232) roll requested 2024-12-09T06:19:03,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143239 2024-12-09T06:19:03,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143239 2024-12-09T06:19:03,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143239) roll requested 2024-12-09T06:19:03,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143247 2024-12-09T06:19:03,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143247 2024-12-09T06:19:03,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143247) roll requested 2024-12-09T06:19:03,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143256 2024-12-09T06:19:03,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143256 2024-12-09T06:19:03,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143256) roll requested 2024-12-09T06:19:03,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143262 2024-12-09T06:19:03,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143262 2024-12-09T06:19:03,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143262) roll requested 2024-12-09T06:19:03,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143269 2024-12-09T06:19:03,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143269 2024-12-09T06:19:03,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143269) roll requested 2024-12-09T06:19:03,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143275 2024-12-09T06:19:03,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143275 2024-12-09T06:19:03,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143275) roll requested 2024-12-09T06:19:03,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143280 2024-12-09T06:19:03,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143280 2024-12-09T06:19:03,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143280) roll requested 2024-12-09T06:19:03,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143286 2024-12-09T06:19:03,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143286 2024-12-09T06:19:03,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143286) roll requested 2024-12-09T06:19:03,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143291 2024-12-09T06:19:03,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143291 2024-12-09T06:19:03,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143291) roll requested 2024-12-09T06:19:03,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143299 2024-12-09T06:19:03,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143299 2024-12-09T06:19:03,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143299) roll requested 2024-12-09T06:19:03,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143306 2024-12-09T06:19:03,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143306 2024-12-09T06:19:03,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143306) roll requested 2024-12-09T06:19:03,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143313 2024-12-09T06:19:03,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143313 2024-12-09T06:19:03,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143313) roll requested 2024-12-09T06:19:03,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143322 2024-12-09T06:19:03,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143322 2024-12-09T06:19:03,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143322) roll requested 2024-12-09T06:19:03,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143330 2024-12-09T06:19:03,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143330 2024-12-09T06:19:03,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143330) roll requested 2024-12-09T06:19:03,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143338 2024-12-09T06:19:03,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143338 2024-12-09T06:19:03,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143338) roll requested 2024-12-09T06:19:03,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143351 2024-12-09T06:19:03,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143351 2024-12-09T06:19:03,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143351) roll requested 2024-12-09T06:19:03,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143357 2024-12-09T06:19:03,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143357 2024-12-09T06:19:03,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143357) roll requested 2024-12-09T06:19:03,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143364 2024-12-09T06:19:03,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143364 2024-12-09T06:19:03,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143364) roll requested 2024-12-09T06:19:03,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143370 2024-12-09T06:19:03,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143370 2024-12-09T06:19:03,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143370) roll requested 2024-12-09T06:19:03,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143377 2024-12-09T06:19:03,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143377 2024-12-09T06:19:03,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143377) roll requested 2024-12-09T06:19:03,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143385 2024-12-09T06:19:03,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143385 2024-12-09T06:19:03,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143385) roll requested 2024-12-09T06:19:03,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143395 2024-12-09T06:19:03,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143395 2024-12-09T06:19:03,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143395) roll requested 2024-12-09T06:19:03,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143404 2024-12-09T06:19:03,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143404 2024-12-09T06:19:03,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143404) roll requested 2024-12-09T06:19:03,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143413 2024-12-09T06:19:03,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143413 2024-12-09T06:19:03,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143413) roll requested 2024-12-09T06:19:03,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143421 2024-12-09T06:19:03,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143421 2024-12-09T06:19:03,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143421) roll requested 2024-12-09T06:19:03,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143429 2024-12-09T06:19:03,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143429 2024-12-09T06:19:03,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143429) roll requested 2024-12-09T06:19:03,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143436 2024-12-09T06:19:03,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143436 2024-12-09T06:19:03,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143436) roll requested 2024-12-09T06:19:03,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143441 2024-12-09T06:19:03,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143441 2024-12-09T06:19:03,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143441) roll requested 2024-12-09T06:19:03,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143447 2024-12-09T06:19:03,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143447 2024-12-09T06:19:03,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143447) roll requested 2024-12-09T06:19:03,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143454 2024-12-09T06:19:03,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143454 2024-12-09T06:19:03,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143454) roll requested 2024-12-09T06:19:03,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143462 2024-12-09T06:19:03,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143462 2024-12-09T06:19:03,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143462) roll requested 2024-12-09T06:19:03,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143469 2024-12-09T06:19:03,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143469 2024-12-09T06:19:03,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143469) roll requested 2024-12-09T06:19:03,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143476 2024-12-09T06:19:03,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143476 2024-12-09T06:19:03,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143476) roll requested 2024-12-09T06:19:03,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143485 2024-12-09T06:19:03,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143485 2024-12-09T06:19:03,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143485) roll requested 2024-12-09T06:19:03,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143493 2024-12-09T06:19:03,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143493 2024-12-09T06:19:03,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143493) roll requested 2024-12-09T06:19:03,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143501 2024-12-09T06:19:03,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143501 2024-12-09T06:19:03,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143501) roll requested 2024-12-09T06:19:03,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143508 2024-12-09T06:19:03,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143508 2024-12-09T06:19:03,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143508) roll requested 2024-12-09T06:19:03,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143521 2024-12-09T06:19:03,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143521 2024-12-09T06:19:03,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143521) roll requested 2024-12-09T06:19:03,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143528 2024-12-09T06:19:03,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143528 2024-12-09T06:19:03,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143528) roll requested 2024-12-09T06:19:03,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143534 2024-12-09T06:19:03,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143534 2024-12-09T06:19:03,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143534) roll requested 2024-12-09T06:19:03,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143539 2024-12-09T06:19:03,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143539 2024-12-09T06:19:03,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143539) roll requested 2024-12-09T06:19:03,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143549 2024-12-09T06:19:03,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143549 2024-12-09T06:19:03,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143549) roll requested 2024-12-09T06:19:03,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143558 2024-12-09T06:19:03,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143558 2024-12-09T06:19:03,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143558) roll requested 2024-12-09T06:19:03,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143565 2024-12-09T06:19:03,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143565 2024-12-09T06:19:03,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143565) roll requested 2024-12-09T06:19:03,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143571 2024-12-09T06:19:03,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143571 2024-12-09T06:19:03,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143571) roll requested 2024-12-09T06:19:03,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143578 2024-12-09T06:19:03,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143578 2024-12-09T06:19:03,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143578) roll requested 2024-12-09T06:19:03,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143585 2024-12-09T06:19:03,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143585 2024-12-09T06:19:03,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143585) roll requested 2024-12-09T06:19:03,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143590 2024-12-09T06:19:03,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143590 2024-12-09T06:19:03,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143590) roll requested 2024-12-09T06:19:03,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143597 2024-12-09T06:19:03,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143597 2024-12-09T06:19:03,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143597) roll requested 2024-12-09T06:19:03,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143603 2024-12-09T06:19:03,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143603 2024-12-09T06:19:03,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143603) roll requested 2024-12-09T06:19:03,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143611 2024-12-09T06:19:03,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143611 2024-12-09T06:19:03,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143611) roll requested 2024-12-09T06:19:03,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143620 2024-12-09T06:19:03,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143620 2024-12-09T06:19:03,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143620) roll requested 2024-12-09T06:19:03,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143629 2024-12-09T06:19:03,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143629 2024-12-09T06:19:03,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143629) roll requested 2024-12-09T06:19:03,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143638 2024-12-09T06:19:03,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143638 2024-12-09T06:19:03,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143638) roll requested 2024-12-09T06:19:03,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143647 2024-12-09T06:19:03,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143647 2024-12-09T06:19:03,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143647) roll requested 2024-12-09T06:19:03,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143656 2024-12-09T06:19:03,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143656 2024-12-09T06:19:03,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143656) roll requested 2024-12-09T06:19:03,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143662 2024-12-09T06:19:03,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143662 2024-12-09T06:19:03,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143662) roll requested 2024-12-09T06:19:03,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143669 2024-12-09T06:19:03,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143669 2024-12-09T06:19:03,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143669) roll requested 2024-12-09T06:19:03,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143677 2024-12-09T06:19:03,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143677 2024-12-09T06:19:03,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143677) roll requested 2024-12-09T06:19:03,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143686 2024-12-09T06:19:03,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143686 2024-12-09T06:19:03,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143686) roll requested 2024-12-09T06:19:03,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143697 2024-12-09T06:19:03,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143697 2024-12-09T06:19:03,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143697) roll requested 2024-12-09T06:19:03,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143704 2024-12-09T06:19:03,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143704 2024-12-09T06:19:03,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143704) roll requested 2024-12-09T06:19:03,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143713 2024-12-09T06:19:03,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143713 2024-12-09T06:19:03,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143713) roll requested 2024-12-09T06:19:03,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143720 2024-12-09T06:19:03,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143720 2024-12-09T06:19:03,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143720) roll requested 2024-12-09T06:19:03,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143728 2024-12-09T06:19:03,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143728 2024-12-09T06:19:03,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143728) roll requested 2024-12-09T06:19:03,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143734 2024-12-09T06:19:03,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143734 2024-12-09T06:19:03,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143734) roll requested 2024-12-09T06:19:03,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143740 2024-12-09T06:19:03,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143740 2024-12-09T06:19:03,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143740) roll requested 2024-12-09T06:19:03,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143745 2024-12-09T06:19:03,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143745 2024-12-09T06:19:03,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143745) roll requested 2024-12-09T06:19:03,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143753 2024-12-09T06:19:03,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143753 2024-12-09T06:19:03,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143753) roll requested 2024-12-09T06:19:03,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143769 2024-12-09T06:19:03,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143769 2024-12-09T06:19:03,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143769) roll requested 2024-12-09T06:19:03,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143776 2024-12-09T06:19:03,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143776 2024-12-09T06:19:03,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143776) roll requested 2024-12-09T06:19:03,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143785 2024-12-09T06:19:03,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143785 2024-12-09T06:19:03,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143785) roll requested 2024-12-09T06:19:03,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143791 2024-12-09T06:19:03,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143791 2024-12-09T06:19:03,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143791) roll requested 2024-12-09T06:19:03,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143804 2024-12-09T06:19:03,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143804 2024-12-09T06:19:03,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143804) roll requested 2024-12-09T06:19:03,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143816 2024-12-09T06:19:03,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143816 2024-12-09T06:19:03,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143816) roll requested 2024-12-09T06:19:03,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143821 2024-12-09T06:19:03,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143821 2024-12-09T06:19:03,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143821) roll requested 2024-12-09T06:19:03,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143828 2024-12-09T06:19:03,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143828 2024-12-09T06:19:03,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143828) roll requested 2024-12-09T06:19:03,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143833 2024-12-09T06:19:03,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143833 2024-12-09T06:19:03,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143833) roll requested 2024-12-09T06:19:03,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143839 2024-12-09T06:19:03,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143839 2024-12-09T06:19:03,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143839) roll requested 2024-12-09T06:19:03,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143845 2024-12-09T06:19:03,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143845 2024-12-09T06:19:03,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143845) roll requested 2024-12-09T06:19:03,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143850 2024-12-09T06:19:03,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143850 2024-12-09T06:19:03,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143850) roll requested 2024-12-09T06:19:03,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143857 2024-12-09T06:19:03,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143857 2024-12-09T06:19:03,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143857) roll requested 2024-12-09T06:19:03,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143864 2024-12-09T06:19:03,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143864 2024-12-09T06:19:03,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143864) roll requested 2024-12-09T06:19:03,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143871 2024-12-09T06:19:03,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143871 2024-12-09T06:19:03,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143871) roll requested 2024-12-09T06:19:03,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143878 2024-12-09T06:19:03,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143878 2024-12-09T06:19:03,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143878) roll requested 2024-12-09T06:19:03,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143884 2024-12-09T06:19:03,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143884 2024-12-09T06:19:03,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143884) roll requested 2024-12-09T06:19:03,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143889 2024-12-09T06:19:03,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143889 2024-12-09T06:19:03,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143889) roll requested 2024-12-09T06:19:03,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143894 2024-12-09T06:19:03,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143894 2024-12-09T06:19:03,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143894) roll requested 2024-12-09T06:19:03,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143900 2024-12-09T06:19:03,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143900 2024-12-09T06:19:03,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143900) roll requested 2024-12-09T06:19:03,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143906 2024-12-09T06:19:03,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143906 2024-12-09T06:19:03,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143906) roll requested 2024-12-09T06:19:03,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143915 2024-12-09T06:19:03,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143915 2024-12-09T06:19:03,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143915) roll requested 2024-12-09T06:19:03,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143924 2024-12-09T06:19:03,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143924 2024-12-09T06:19:03,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143924) roll requested 2024-12-09T06:19:03,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143933 2024-12-09T06:19:03,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143933 2024-12-09T06:19:03,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143933) roll requested 2024-12-09T06:19:03,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143944 2024-12-09T06:19:03,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143944 2024-12-09T06:19:03,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143944) roll requested 2024-12-09T06:19:03,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143953 2024-12-09T06:19:03,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143953 2024-12-09T06:19:03,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143953) roll requested 2024-12-09T06:19:03,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143961 2024-12-09T06:19:03,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143961 2024-12-09T06:19:03,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143961) roll requested 2024-12-09T06:19:03,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143973 2024-12-09T06:19:03,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143973 2024-12-09T06:19:03,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143973) roll requested 2024-12-09T06:19:03,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143981 2024-12-09T06:19:03,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143981 2024-12-09T06:19:03,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143981) roll requested 2024-12-09T06:19:03,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143988 2024-12-09T06:19:03,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:03,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143988 2024-12-09T06:19:03,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:03,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:03,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143988) roll requested 2024-12-09T06:19:03,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725143996 2024-12-09T06:19:04,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143996 2024-12-09T06:19:04,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725143996) roll requested 2024-12-09T06:19:04,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144007 2024-12-09T06:19:04,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725143996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144007 2024-12-09T06:19:04,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144007) roll requested 2024-12-09T06:19:04,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144016 2024-12-09T06:19:04,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144016 2024-12-09T06:19:04,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144016) roll requested 2024-12-09T06:19:04,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144022 2024-12-09T06:19:04,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144022 2024-12-09T06:19:04,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144022) roll requested 2024-12-09T06:19:04,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144029 2024-12-09T06:19:04,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144029 2024-12-09T06:19:04,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144029) roll requested 2024-12-09T06:19:04,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144036 2024-12-09T06:19:04,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144036 2024-12-09T06:19:04,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144036) roll requested 2024-12-09T06:19:04,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144042 2024-12-09T06:19:04,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144042 2024-12-09T06:19:04,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144042) roll requested 2024-12-09T06:19:04,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144050 2024-12-09T06:19:04,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144050 2024-12-09T06:19:04,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144050) roll requested 2024-12-09T06:19:04,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144059 2024-12-09T06:19:04,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144059 2024-12-09T06:19:04,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144059) roll requested 2024-12-09T06:19:04,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144068 2024-12-09T06:19:04,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144068 2024-12-09T06:19:04,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144068) roll requested 2024-12-09T06:19:04,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144077 2024-12-09T06:19:04,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144077 2024-12-09T06:19:04,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144077) roll requested 2024-12-09T06:19:04,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144087 2024-12-09T06:19:04,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144087 2024-12-09T06:19:04,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144087) roll requested 2024-12-09T06:19:04,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144094 2024-12-09T06:19:04,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144094 2024-12-09T06:19:04,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144094) roll requested 2024-12-09T06:19:04,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144103 2024-12-09T06:19:04,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144103 2024-12-09T06:19:04,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144103) roll requested 2024-12-09T06:19:04,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144112 2024-12-09T06:19:04,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144112 2024-12-09T06:19:04,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144112) roll requested 2024-12-09T06:19:04,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144121 2024-12-09T06:19:04,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144121 2024-12-09T06:19:04,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144121) roll requested 2024-12-09T06:19:04,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144131 2024-12-09T06:19:04,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144131 2024-12-09T06:19:04,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144131) roll requested 2024-12-09T06:19:04,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144140 2024-12-09T06:19:04,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144140 2024-12-09T06:19:04,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144140) roll requested 2024-12-09T06:19:04,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144150 2024-12-09T06:19:04,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144150 2024-12-09T06:19:04,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144150) roll requested 2024-12-09T06:19:04,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144157 2024-12-09T06:19:04,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144157 2024-12-09T06:19:04,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144157) roll requested 2024-12-09T06:19:04,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144164 2024-12-09T06:19:04,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144164 2024-12-09T06:19:04,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144164) roll requested 2024-12-09T06:19:04,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144172 2024-12-09T06:19:04,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144172 2024-12-09T06:19:04,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144172) roll requested 2024-12-09T06:19:04,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144181 2024-12-09T06:19:04,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144181 2024-12-09T06:19:04,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144181) roll requested 2024-12-09T06:19:04,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144187 2024-12-09T06:19:04,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144187 2024-12-09T06:19:04,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144187) roll requested 2024-12-09T06:19:04,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144193 2024-12-09T06:19:04,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144193 2024-12-09T06:19:04,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144193) roll requested 2024-12-09T06:19:04,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144200 2024-12-09T06:19:04,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144200 2024-12-09T06:19:04,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144200) roll requested 2024-12-09T06:19:04,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144208 2024-12-09T06:19:04,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144208 2024-12-09T06:19:04,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144208) roll requested 2024-12-09T06:19:04,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144217 2024-12-09T06:19:04,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144217 2024-12-09T06:19:04,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144217) roll requested 2024-12-09T06:19:04,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144225 2024-12-09T06:19:04,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144225 2024-12-09T06:19:04,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144225) roll requested 2024-12-09T06:19:04,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144234 2024-12-09T06:19:04,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144234 2024-12-09T06:19:04,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144234) roll requested 2024-12-09T06:19:04,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144244 2024-12-09T06:19:04,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144244 2024-12-09T06:19:04,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144244) roll requested 2024-12-09T06:19:04,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144254 2024-12-09T06:19:04,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144254 2024-12-09T06:19:04,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144254) roll requested 2024-12-09T06:19:04,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144263 2024-12-09T06:19:04,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144263 2024-12-09T06:19:04,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144263) roll requested 2024-12-09T06:19:04,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144271 2024-12-09T06:19:04,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144271 2024-12-09T06:19:04,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144271) roll requested 2024-12-09T06:19:04,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144279 2024-12-09T06:19:04,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144279 2024-12-09T06:19:04,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144279) roll requested 2024-12-09T06:19:04,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144288 2024-12-09T06:19:04,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144288 2024-12-09T06:19:04,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144288) roll requested 2024-12-09T06:19:04,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144295 2024-12-09T06:19:04,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144295 2024-12-09T06:19:04,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144295) roll requested 2024-12-09T06:19:04,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144301 2024-12-09T06:19:04,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144301 2024-12-09T06:19:04,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144301) roll requested 2024-12-09T06:19:04,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144309 2024-12-09T06:19:04,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144309 2024-12-09T06:19:04,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144309) roll requested 2024-12-09T06:19:04,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144317 2024-12-09T06:19:04,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144317 2024-12-09T06:19:04,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144317) roll requested 2024-12-09T06:19:04,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144323 2024-12-09T06:19:04,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144323 2024-12-09T06:19:04,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144323) roll requested 2024-12-09T06:19:04,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144329 2024-12-09T06:19:04,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144329 2024-12-09T06:19:04,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144329) roll requested 2024-12-09T06:19:04,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144336 2024-12-09T06:19:04,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144336 2024-12-09T06:19:04,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144336) roll requested 2024-12-09T06:19:04,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144344 2024-12-09T06:19:04,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144344 2024-12-09T06:19:04,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144344) roll requested 2024-12-09T06:19:04,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144352 2024-12-09T06:19:04,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144352 2024-12-09T06:19:04,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144352) roll requested 2024-12-09T06:19:04,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144360 2024-12-09T06:19:04,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144360 2024-12-09T06:19:04,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144360) roll requested 2024-12-09T06:19:04,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144366 2024-12-09T06:19:04,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144366 2024-12-09T06:19:04,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144366) roll requested 2024-12-09T06:19:04,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144372 2024-12-09T06:19:04,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144372 2024-12-09T06:19:04,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144372) roll requested 2024-12-09T06:19:04,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144379 2024-12-09T06:19:04,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144379 2024-12-09T06:19:04,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144379) roll requested 2024-12-09T06:19:04,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144387 2024-12-09T06:19:04,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144387 2024-12-09T06:19:04,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144387) roll requested 2024-12-09T06:19:04,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144402 2024-12-09T06:19:04,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144402 2024-12-09T06:19:04,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144402) roll requested 2024-12-09T06:19:04,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144411 2024-12-09T06:19:04,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144411 2024-12-09T06:19:04,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144411) roll requested 2024-12-09T06:19:04,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144421 2024-12-09T06:19:04,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144421 2024-12-09T06:19:04,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144421) roll requested 2024-12-09T06:19:04,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144429 2024-12-09T06:19:04,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144429 2024-12-09T06:19:04,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144429) roll requested 2024-12-09T06:19:04,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144436 2024-12-09T06:19:04,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144436 2024-12-09T06:19:04,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144436) roll requested 2024-12-09T06:19:04,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144442 2024-12-09T06:19:04,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144442 2024-12-09T06:19:04,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144442) roll requested 2024-12-09T06:19:04,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144448 2024-12-09T06:19:04,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144448 2024-12-09T06:19:04,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144448) roll requested 2024-12-09T06:19:04,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144455 2024-12-09T06:19:04,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144455 2024-12-09T06:19:04,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144455) roll requested 2024-12-09T06:19:04,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144461 2024-12-09T06:19:04,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144461 2024-12-09T06:19:04,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144461) roll requested 2024-12-09T06:19:04,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144467 2024-12-09T06:19:04,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144467 2024-12-09T06:19:04,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144467) roll requested 2024-12-09T06:19:04,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144476 2024-12-09T06:19:04,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144476 2024-12-09T06:19:04,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144476) roll requested 2024-12-09T06:19:04,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144490 2024-12-09T06:19:04,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144490 2024-12-09T06:19:04,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144490) roll requested 2024-12-09T06:19:04,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144504 2024-12-09T06:19:04,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144504 2024-12-09T06:19:04,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144504) roll requested 2024-12-09T06:19:04,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144514 2024-12-09T06:19:04,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144514 2024-12-09T06:19:04,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144514) roll requested 2024-12-09T06:19:04,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144524 2024-12-09T06:19:04,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144524 2024-12-09T06:19:04,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144524) roll requested 2024-12-09T06:19:04,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144530 2024-12-09T06:19:04,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144530 2024-12-09T06:19:04,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144530) roll requested 2024-12-09T06:19:04,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144536 2024-12-09T06:19:04,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144536 2024-12-09T06:19:04,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144536) roll requested 2024-12-09T06:19:04,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144542 2024-12-09T06:19:04,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144542 2024-12-09T06:19:04,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144542) roll requested 2024-12-09T06:19:04,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144552 2024-12-09T06:19:04,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144552 2024-12-09T06:19:04,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144552) roll requested 2024-12-09T06:19:04,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144558 2024-12-09T06:19:04,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144558 2024-12-09T06:19:04,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144558) roll requested 2024-12-09T06:19:04,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144566 2024-12-09T06:19:04,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144566 2024-12-09T06:19:04,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144566) roll requested 2024-12-09T06:19:04,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144572 2024-12-09T06:19:04,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144572 2024-12-09T06:19:04,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144572) roll requested 2024-12-09T06:19:04,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144578 2024-12-09T06:19:04,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144578 2024-12-09T06:19:04,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144578) roll requested 2024-12-09T06:19:04,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144583 2024-12-09T06:19:04,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144583 2024-12-09T06:19:04,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144583) roll requested 2024-12-09T06:19:04,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144591 2024-12-09T06:19:04,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144591 2024-12-09T06:19:04,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144591) roll requested 2024-12-09T06:19:04,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144597 2024-12-09T06:19:04,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144597 2024-12-09T06:19:04,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144597) roll requested 2024-12-09T06:19:04,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144607 2024-12-09T06:19:04,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144607 2024-12-09T06:19:04,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144607) roll requested 2024-12-09T06:19:04,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144616 2024-12-09T06:19:04,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144616 2024-12-09T06:19:04,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144616) roll requested 2024-12-09T06:19:04,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144622 2024-12-09T06:19:04,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144622 2024-12-09T06:19:04,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144622) roll requested 2024-12-09T06:19:04,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144628 2024-12-09T06:19:04,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144628 2024-12-09T06:19:04,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144628) roll requested 2024-12-09T06:19:04,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144634 2024-12-09T06:19:04,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144634 2024-12-09T06:19:04,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144634) roll requested 2024-12-09T06:19:04,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144641 2024-12-09T06:19:04,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144641 2024-12-09T06:19:04,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144641) roll requested 2024-12-09T06:19:04,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144647 2024-12-09T06:19:04,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144647 2024-12-09T06:19:04,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144647) roll requested 2024-12-09T06:19:04,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144657 2024-12-09T06:19:04,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144657 2024-12-09T06:19:04,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144657) roll requested 2024-12-09T06:19:04,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144664 2024-12-09T06:19:04,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144664 2024-12-09T06:19:04,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144664) roll requested 2024-12-09T06:19:04,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144669 2024-12-09T06:19:04,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144669 2024-12-09T06:19:04,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144669) roll requested 2024-12-09T06:19:04,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144675 2024-12-09T06:19:04,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144675 2024-12-09T06:19:04,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144675) roll requested 2024-12-09T06:19:04,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144683 2024-12-09T06:19:04,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144683 2024-12-09T06:19:04,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144683) roll requested 2024-12-09T06:19:04,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144690 2024-12-09T06:19:04,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144690 2024-12-09T06:19:04,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144690) roll requested 2024-12-09T06:19:04,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144699 2024-12-09T06:19:04,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144699 2024-12-09T06:19:04,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144699) roll requested 2024-12-09T06:19:04,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144709 2024-12-09T06:19:04,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144709 2024-12-09T06:19:04,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144709) roll requested 2024-12-09T06:19:04,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144714 2024-12-09T06:19:04,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144714 2024-12-09T06:19:04,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144714) roll requested 2024-12-09T06:19:04,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144720 2024-12-09T06:19:04,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144720 2024-12-09T06:19:04,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144720) roll requested 2024-12-09T06:19:04,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144727 2024-12-09T06:19:04,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144727 2024-12-09T06:19:04,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144727) roll requested 2024-12-09T06:19:04,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144736 2024-12-09T06:19:04,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144736 2024-12-09T06:19:04,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144736) roll requested 2024-12-09T06:19:04,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144745 2024-12-09T06:19:04,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144745 2024-12-09T06:19:04,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144745) roll requested 2024-12-09T06:19:04,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144750 2024-12-09T06:19:04,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144750 2024-12-09T06:19:04,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144750) roll requested 2024-12-09T06:19:04,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144756 2024-12-09T06:19:04,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144756 2024-12-09T06:19:04,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144756) roll requested 2024-12-09T06:19:04,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144763 2024-12-09T06:19:04,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144763 2024-12-09T06:19:04,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144763) roll requested 2024-12-09T06:19:04,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144771 2024-12-09T06:19:04,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144771 2024-12-09T06:19:04,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144771) roll requested 2024-12-09T06:19:04,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144780 2024-12-09T06:19:04,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144780 2024-12-09T06:19:04,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144780) roll requested 2024-12-09T06:19:04,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144789 2024-12-09T06:19:04,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144789 2024-12-09T06:19:04,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144789) roll requested 2024-12-09T06:19:04,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144796 2024-12-09T06:19:04,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144796 2024-12-09T06:19:04,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144796) roll requested 2024-12-09T06:19:04,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144803 2024-12-09T06:19:04,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144803 2024-12-09T06:19:04,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144803) roll requested 2024-12-09T06:19:04,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144809 2024-12-09T06:19:04,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144809 2024-12-09T06:19:04,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144809) roll requested 2024-12-09T06:19:04,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144814 2024-12-09T06:19:04,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144814 2024-12-09T06:19:04,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144814) roll requested 2024-12-09T06:19:04,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144822 2024-12-09T06:19:04,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144822 2024-12-09T06:19:04,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144822) roll requested 2024-12-09T06:19:04,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144828 2024-12-09T06:19:04,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144828 2024-12-09T06:19:04,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144828) roll requested 2024-12-09T06:19:04,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144833 2024-12-09T06:19:04,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144833 2024-12-09T06:19:04,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144833) roll requested 2024-12-09T06:19:04,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144838 2024-12-09T06:19:04,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144838 2024-12-09T06:19:04,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144838) roll requested 2024-12-09T06:19:04,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144844 2024-12-09T06:19:04,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144844 2024-12-09T06:19:04,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144844) roll requested 2024-12-09T06:19:04,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144851 2024-12-09T06:19:04,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144851 2024-12-09T06:19:04,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144851) roll requested 2024-12-09T06:19:04,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144862 2024-12-09T06:19:04,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144862 2024-12-09T06:19:04,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144862) roll requested 2024-12-09T06:19:04,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144873 2024-12-09T06:19:04,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144873 2024-12-09T06:19:04,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144873) roll requested 2024-12-09T06:19:04,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144881 2024-12-09T06:19:04,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144881 2024-12-09T06:19:04,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144881) roll requested 2024-12-09T06:19:04,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144891 2024-12-09T06:19:04,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144891 2024-12-09T06:19:04,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144891) roll requested 2024-12-09T06:19:04,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144897 2024-12-09T06:19:04,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144897 2024-12-09T06:19:04,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144897) roll requested 2024-12-09T06:19:04,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144904 2024-12-09T06:19:04,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144904 2024-12-09T06:19:04,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144904) roll requested 2024-12-09T06:19:04,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144919 2024-12-09T06:19:04,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144919 2024-12-09T06:19:04,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144919) roll requested 2024-12-09T06:19:04,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144929 2024-12-09T06:19:04,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144929 2024-12-09T06:19:04,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144929) roll requested 2024-12-09T06:19:04,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144935 2024-12-09T06:19:04,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144935 2024-12-09T06:19:04,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144935) roll requested 2024-12-09T06:19:04,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144940 2024-12-09T06:19:04,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144940 2024-12-09T06:19:04,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144940) roll requested 2024-12-09T06:19:04,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144946 2024-12-09T06:19:04,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144946 2024-12-09T06:19:04,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144946) roll requested 2024-12-09T06:19:04,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144952 2024-12-09T06:19:04,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144952 2024-12-09T06:19:04,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144952) roll requested 2024-12-09T06:19:04,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144962 2024-12-09T06:19:04,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144962 2024-12-09T06:19:04,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144962) roll requested 2024-12-09T06:19:04,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144967 2024-12-09T06:19:04,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144967 2024-12-09T06:19:04,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144967) roll requested 2024-12-09T06:19:04,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144972 2024-12-09T06:19:04,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144972 2024-12-09T06:19:04,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:04,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:04,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144972) roll requested 2024-12-09T06:19:04,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725144981 2024-12-09T06:19:04,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:04,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144981 2024-12-09T06:19:05,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725144981) roll requested 2024-12-09T06:19:05,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145006 2024-12-09T06:19:05,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725144981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145006 2024-12-09T06:19:05,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145006) roll requested 2024-12-09T06:19:05,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145011 2024-12-09T06:19:05,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145011 2024-12-09T06:19:05,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145011) roll requested 2024-12-09T06:19:05,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145016 2024-12-09T06:19:05,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145016 2024-12-09T06:19:05,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145016) roll requested 2024-12-09T06:19:05,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145023 2024-12-09T06:19:05,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145023 2024-12-09T06:19:05,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145023) roll requested 2024-12-09T06:19:05,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145029 2024-12-09T06:19:05,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145029 2024-12-09T06:19:05,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145029) roll requested 2024-12-09T06:19:05,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145037 2024-12-09T06:19:05,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145037 2024-12-09T06:19:05,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145037) roll requested 2024-12-09T06:19:05,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145044 2024-12-09T06:19:05,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145044 2024-12-09T06:19:05,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145044) roll requested 2024-12-09T06:19:05,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145052 2024-12-09T06:19:05,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145052 2024-12-09T06:19:05,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145052) roll requested 2024-12-09T06:19:05,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145059 2024-12-09T06:19:05,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145059 2024-12-09T06:19:05,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145059) roll requested 2024-12-09T06:19:05,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145068 2024-12-09T06:19:05,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145068 2024-12-09T06:19:05,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145068) roll requested 2024-12-09T06:19:05,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145074 2024-12-09T06:19:05,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145074 2024-12-09T06:19:05,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145074) roll requested 2024-12-09T06:19:05,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145079 2024-12-09T06:19:05,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145079 2024-12-09T06:19:05,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145079) roll requested 2024-12-09T06:19:05,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145085 2024-12-09T06:19:05,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145085 2024-12-09T06:19:05,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145085) roll requested 2024-12-09T06:19:05,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145092 2024-12-09T06:19:05,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145092 2024-12-09T06:19:05,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145092) roll requested 2024-12-09T06:19:05,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145098 2024-12-09T06:19:05,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145098 2024-12-09T06:19:05,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145098) roll requested 2024-12-09T06:19:05,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145104 2024-12-09T06:19:05,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145104 2024-12-09T06:19:05,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145104) roll requested 2024-12-09T06:19:05,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145112 2024-12-09T06:19:05,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145112 2024-12-09T06:19:05,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145112) roll requested 2024-12-09T06:19:05,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145121 2024-12-09T06:19:05,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145121 2024-12-09T06:19:05,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145121) roll requested 2024-12-09T06:19:05,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145131 2024-12-09T06:19:05,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145131 2024-12-09T06:19:05,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145131) roll requested 2024-12-09T06:19:05,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145140 2024-12-09T06:19:05,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145140 2024-12-09T06:19:05,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145140) roll requested 2024-12-09T06:19:05,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145149 2024-12-09T06:19:05,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145149 2024-12-09T06:19:05,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145149) roll requested 2024-12-09T06:19:05,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145158 2024-12-09T06:19:05,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145158 2024-12-09T06:19:05,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145158) roll requested 2024-12-09T06:19:05,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145168 2024-12-09T06:19:05,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145168 2024-12-09T06:19:05,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145168) roll requested 2024-12-09T06:19:05,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145177 2024-12-09T06:19:05,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145177 2024-12-09T06:19:05,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145177) roll requested 2024-12-09T06:19:05,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145186 2024-12-09T06:19:05,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145186 2024-12-09T06:19:05,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145186) roll requested 2024-12-09T06:19:05,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145194 2024-12-09T06:19:05,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145194 2024-12-09T06:19:05,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145194) roll requested 2024-12-09T06:19:05,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145204 2024-12-09T06:19:05,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145204 2024-12-09T06:19:05,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145204) roll requested 2024-12-09T06:19:05,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145209 2024-12-09T06:19:05,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145209 2024-12-09T06:19:05,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145209) roll requested 2024-12-09T06:19:05,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145215 2024-12-09T06:19:05,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145215 2024-12-09T06:19:05,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145215) roll requested 2024-12-09T06:19:05,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145221 2024-12-09T06:19:05,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145221 2024-12-09T06:19:05,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145221) roll requested 2024-12-09T06:19:05,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145228 2024-12-09T06:19:05,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145228 2024-12-09T06:19:05,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145228) roll requested 2024-12-09T06:19:05,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145235 2024-12-09T06:19:05,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145235 2024-12-09T06:19:05,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145235) roll requested 2024-12-09T06:19:05,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145245 2024-12-09T06:19:05,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145245 2024-12-09T06:19:05,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145245) roll requested 2024-12-09T06:19:05,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145254 2024-12-09T06:19:05,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145254 2024-12-09T06:19:05,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145254) roll requested 2024-12-09T06:19:05,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145262 2024-12-09T06:19:05,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145262 2024-12-09T06:19:05,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145262) roll requested 2024-12-09T06:19:05,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145269 2024-12-09T06:19:05,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145269 2024-12-09T06:19:05,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145269) roll requested 2024-12-09T06:19:05,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145275 2024-12-09T06:19:05,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145275 2024-12-09T06:19:05,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145275) roll requested 2024-12-09T06:19:05,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145281 2024-12-09T06:19:05,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145281 2024-12-09T06:19:05,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145281) roll requested 2024-12-09T06:19:05,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145289 2024-12-09T06:19:05,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145289 2024-12-09T06:19:05,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145289) roll requested 2024-12-09T06:19:05,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145296 2024-12-09T06:19:05,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145296 2024-12-09T06:19:05,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145296) roll requested 2024-12-09T06:19:05,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145301 2024-12-09T06:19:05,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145301 2024-12-09T06:19:05,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145301) roll requested 2024-12-09T06:19:05,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145309 2024-12-09T06:19:05,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145309 2024-12-09T06:19:05,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145309) roll requested 2024-12-09T06:19:05,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145317 2024-12-09T06:19:05,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145317 2024-12-09T06:19:05,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145317) roll requested 2024-12-09T06:19:05,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145325 2024-12-09T06:19:05,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145325 2024-12-09T06:19:05,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145325) roll requested 2024-12-09T06:19:05,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145333 2024-12-09T06:19:05,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145333 2024-12-09T06:19:05,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145333) roll requested 2024-12-09T06:19:05,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145339 2024-12-09T06:19:05,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145339 2024-12-09T06:19:05,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145339) roll requested 2024-12-09T06:19:05,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145345 2024-12-09T06:19:05,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145345 2024-12-09T06:19:05,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145345) roll requested 2024-12-09T06:19:05,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145351 2024-12-09T06:19:05,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145351 2024-12-09T06:19:05,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145351) roll requested 2024-12-09T06:19:05,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145358 2024-12-09T06:19:05,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145358 2024-12-09T06:19:05,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145358) roll requested 2024-12-09T06:19:05,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145364 2024-12-09T06:19:05,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145364 2024-12-09T06:19:05,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145364) roll requested 2024-12-09T06:19:05,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145369 2024-12-09T06:19:05,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145369 2024-12-09T06:19:05,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145369) roll requested 2024-12-09T06:19:05,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145375 2024-12-09T06:19:05,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145375 2024-12-09T06:19:05,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145375) roll requested 2024-12-09T06:19:05,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145379 2024-12-09T06:19:05,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145379 2024-12-09T06:19:05,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145379) roll requested 2024-12-09T06:19:05,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145385 2024-12-09T06:19:05,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145385 2024-12-09T06:19:05,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145385) roll requested 2024-12-09T06:19:05,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145392 2024-12-09T06:19:05,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145392 2024-12-09T06:19:05,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145392) roll requested 2024-12-09T06:19:05,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145399 2024-12-09T06:19:05,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145399 2024-12-09T06:19:05,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145399) roll requested 2024-12-09T06:19:05,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145405 2024-12-09T06:19:05,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145405 2024-12-09T06:19:05,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145405) roll requested 2024-12-09T06:19:05,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145413 2024-12-09T06:19:05,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145413 2024-12-09T06:19:05,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145413) roll requested 2024-12-09T06:19:05,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145418 2024-12-09T06:19:05,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145418 2024-12-09T06:19:05,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145418) roll requested 2024-12-09T06:19:05,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145425 2024-12-09T06:19:05,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145425 2024-12-09T06:19:05,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145425) roll requested 2024-12-09T06:19:05,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145430 2024-12-09T06:19:05,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145430 2024-12-09T06:19:05,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145430) roll requested 2024-12-09T06:19:05,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145436 2024-12-09T06:19:05,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145436 2024-12-09T06:19:05,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145436) roll requested 2024-12-09T06:19:05,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145442 2024-12-09T06:19:05,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145442 2024-12-09T06:19:05,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145442) roll requested 2024-12-09T06:19:05,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145447 2024-12-09T06:19:05,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145447 2024-12-09T06:19:05,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145447) roll requested 2024-12-09T06:19:05,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145454 2024-12-09T06:19:05,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145454 2024-12-09T06:19:05,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145454) roll requested 2024-12-09T06:19:05,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145461 2024-12-09T06:19:05,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145461 2024-12-09T06:19:05,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145461) roll requested 2024-12-09T06:19:05,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145470 2024-12-09T06:19:05,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145470 2024-12-09T06:19:05,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145470) roll requested 2024-12-09T06:19:05,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145484 2024-12-09T06:19:05,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145484 2024-12-09T06:19:05,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145484) roll requested 2024-12-09T06:19:05,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145497 2024-12-09T06:19:05,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145497 2024-12-09T06:19:05,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145497) roll requested 2024-12-09T06:19:05,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145504 2024-12-09T06:19:05,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145504 2024-12-09T06:19:05,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145504) roll requested 2024-12-09T06:19:05,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145509 2024-12-09T06:19:05,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145509 2024-12-09T06:19:05,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145509) roll requested 2024-12-09T06:19:05,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145514 2024-12-09T06:19:05,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145514 2024-12-09T06:19:05,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145514) roll requested 2024-12-09T06:19:05,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145519 2024-12-09T06:19:05,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145519 2024-12-09T06:19:05,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145519) roll requested 2024-12-09T06:19:05,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145527 2024-12-09T06:19:05,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145527 2024-12-09T06:19:05,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145527) roll requested 2024-12-09T06:19:05,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145535 2024-12-09T06:19:05,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145535 2024-12-09T06:19:05,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145535) roll requested 2024-12-09T06:19:05,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145540 2024-12-09T06:19:05,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145540 2024-12-09T06:19:05,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145540) roll requested 2024-12-09T06:19:05,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145545 2024-12-09T06:19:05,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145545 2024-12-09T06:19:05,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145545) roll requested 2024-12-09T06:19:05,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145553 2024-12-09T06:19:05,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145553 2024-12-09T06:19:05,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145553) roll requested 2024-12-09T06:19:05,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145559 2024-12-09T06:19:05,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145559 2024-12-09T06:19:05,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145559) roll requested 2024-12-09T06:19:05,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145568 2024-12-09T06:19:05,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145568 2024-12-09T06:19:05,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145568) roll requested 2024-12-09T06:19:05,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145575 2024-12-09T06:19:05,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145575 2024-12-09T06:19:05,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145575) roll requested 2024-12-09T06:19:05,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145585 2024-12-09T06:19:05,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145585 2024-12-09T06:19:05,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145585) roll requested 2024-12-09T06:19:05,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145594 2024-12-09T06:19:05,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145594 2024-12-09T06:19:05,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145594) roll requested 2024-12-09T06:19:05,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145605 2024-12-09T06:19:05,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145605 2024-12-09T06:19:05,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145605) roll requested 2024-12-09T06:19:05,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145615 2024-12-09T06:19:05,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145615 2024-12-09T06:19:05,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145615) roll requested 2024-12-09T06:19:05,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145620 2024-12-09T06:19:05,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145620 2024-12-09T06:19:05,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145620) roll requested 2024-12-09T06:19:05,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145625 2024-12-09T06:19:05,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145625 2024-12-09T06:19:05,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145625) roll requested 2024-12-09T06:19:05,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145632 2024-12-09T06:19:05,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145632 2024-12-09T06:19:05,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145632) roll requested 2024-12-09T06:19:05,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145639 2024-12-09T06:19:05,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145639 2024-12-09T06:19:05,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145639) roll requested 2024-12-09T06:19:05,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145647 2024-12-09T06:19:05,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145647 2024-12-09T06:19:05,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145647) roll requested 2024-12-09T06:19:05,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145657 2024-12-09T06:19:05,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145657 2024-12-09T06:19:05,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145657) roll requested 2024-12-09T06:19:05,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145665 2024-12-09T06:19:05,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145665 2024-12-09T06:19:05,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145665) roll requested 2024-12-09T06:19:05,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145672 2024-12-09T06:19:05,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145672 2024-12-09T06:19:05,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145672) roll requested 2024-12-09T06:19:05,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145677 2024-12-09T06:19:05,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145677 2024-12-09T06:19:05,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145677) roll requested 2024-12-09T06:19:05,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145682 2024-12-09T06:19:05,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145682 2024-12-09T06:19:05,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145682) roll requested 2024-12-09T06:19:05,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145689 2024-12-09T06:19:05,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145689 2024-12-09T06:19:05,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145689) roll requested 2024-12-09T06:19:05,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145697 2024-12-09T06:19:05,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145697 2024-12-09T06:19:05,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145697) roll requested 2024-12-09T06:19:05,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145705 2024-12-09T06:19:05,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145705 2024-12-09T06:19:05,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145705) roll requested 2024-12-09T06:19:05,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145712 2024-12-09T06:19:05,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145712 2024-12-09T06:19:05,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145712) roll requested 2024-12-09T06:19:05,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145718 2024-12-09T06:19:05,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145718 2024-12-09T06:19:05,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145718) roll requested 2024-12-09T06:19:05,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145725 2024-12-09T06:19:05,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145725 2024-12-09T06:19:05,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145725) roll requested 2024-12-09T06:19:05,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145733 2024-12-09T06:19:05,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145733 2024-12-09T06:19:05,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145733) roll requested 2024-12-09T06:19:05,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145742 2024-12-09T06:19:05,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145742 2024-12-09T06:19:05,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145742) roll requested 2024-12-09T06:19:05,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145749 2024-12-09T06:19:05,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145749 2024-12-09T06:19:05,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145749) roll requested 2024-12-09T06:19:05,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145755 2024-12-09T06:19:05,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145755 2024-12-09T06:19:05,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145755) roll requested 2024-12-09T06:19:05,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145760 2024-12-09T06:19:05,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145760 2024-12-09T06:19:05,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145760) roll requested 2024-12-09T06:19:05,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145766 2024-12-09T06:19:05,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145766 2024-12-09T06:19:05,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145766) roll requested 2024-12-09T06:19:05,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145773 2024-12-09T06:19:05,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145773 2024-12-09T06:19:05,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145773) roll requested 2024-12-09T06:19:05,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145780 2024-12-09T06:19:05,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145780 2024-12-09T06:19:05,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145780) roll requested 2024-12-09T06:19:05,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145788 2024-12-09T06:19:05,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145788 2024-12-09T06:19:05,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145788) roll requested 2024-12-09T06:19:05,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145797 2024-12-09T06:19:05,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145797 2024-12-09T06:19:05,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145797) roll requested 2024-12-09T06:19:05,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145806 2024-12-09T06:19:05,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145806 2024-12-09T06:19:05,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145806) roll requested 2024-12-09T06:19:05,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145816 2024-12-09T06:19:05,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145816 2024-12-09T06:19:05,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145816) roll requested 2024-12-09T06:19:05,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145826 2024-12-09T06:19:05,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145826 2024-12-09T06:19:05,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145826) roll requested 2024-12-09T06:19:05,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145833 2024-12-09T06:19:05,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145833 2024-12-09T06:19:05,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145833) roll requested 2024-12-09T06:19:05,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145841 2024-12-09T06:19:05,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145841 2024-12-09T06:19:05,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145841) roll requested 2024-12-09T06:19:05,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145849 2024-12-09T06:19:05,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145849 2024-12-09T06:19:05,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145849) roll requested 2024-12-09T06:19:05,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145857 2024-12-09T06:19:05,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145857 2024-12-09T06:19:05,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145857) roll requested 2024-12-09T06:19:05,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145867 2024-12-09T06:19:05,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145867 2024-12-09T06:19:05,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145867) roll requested 2024-12-09T06:19:05,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145876 2024-12-09T06:19:05,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145876 2024-12-09T06:19:05,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145876) roll requested 2024-12-09T06:19:05,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145889 2024-12-09T06:19:05,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145889 2024-12-09T06:19:05,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145889) roll requested 2024-12-09T06:19:05,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145902 2024-12-09T06:19:05,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145902 2024-12-09T06:19:05,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145902) roll requested 2024-12-09T06:19:05,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145908 2024-12-09T06:19:05,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145908 2024-12-09T06:19:05,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145908) roll requested 2024-12-09T06:19:05,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145913 2024-12-09T06:19:05,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145913 2024-12-09T06:19:05,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145913) roll requested 2024-12-09T06:19:05,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145920 2024-12-09T06:19:05,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145920 2024-12-09T06:19:05,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145920) roll requested 2024-12-09T06:19:05,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145929 2024-12-09T06:19:05,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145929 2024-12-09T06:19:05,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145929) roll requested 2024-12-09T06:19:05,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145935 2024-12-09T06:19:05,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145935 2024-12-09T06:19:05,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145935) roll requested 2024-12-09T06:19:05,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145941 2024-12-09T06:19:05,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145941 2024-12-09T06:19:05,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145941) roll requested 2024-12-09T06:19:05,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145947 2024-12-09T06:19:05,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145947 2024-12-09T06:19:05,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145947) roll requested 2024-12-09T06:19:05,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145953 2024-12-09T06:19:05,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145953 2024-12-09T06:19:05,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145953) roll requested 2024-12-09T06:19:05,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145958 2024-12-09T06:19:05,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145958 2024-12-09T06:19:05,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145958) roll requested 2024-12-09T06:19:05,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145963 2024-12-09T06:19:05,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145963 2024-12-09T06:19:05,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145963) roll requested 2024-12-09T06:19:05,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145969 2024-12-09T06:19:05,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145969 2024-12-09T06:19:05,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145969) roll requested 2024-12-09T06:19:05,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145977 2024-12-09T06:19:05,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145977 2024-12-09T06:19:05,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145977) roll requested 2024-12-09T06:19:05,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145984 2024-12-09T06:19:05,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145984 2024-12-09T06:19:05,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:05,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:05,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145984) roll requested 2024-12-09T06:19:05,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725145992 2024-12-09T06:19:05,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:05,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145992 2024-12-09T06:19:06,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725145992) roll requested 2024-12-09T06:19:06,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146001 2024-12-09T06:19:06,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725145992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146001 2024-12-09T06:19:06,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146001) roll requested 2024-12-09T06:19:06,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146009 2024-12-09T06:19:06,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146009 2024-12-09T06:19:06,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146009) roll requested 2024-12-09T06:19:06,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146014 2024-12-09T06:19:06,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146014 2024-12-09T06:19:06,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146014) roll requested 2024-12-09T06:19:06,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146020 2024-12-09T06:19:06,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146020 2024-12-09T06:19:06,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146020) roll requested 2024-12-09T06:19:06,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146028 2024-12-09T06:19:06,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146028 2024-12-09T06:19:06,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146028) roll requested 2024-12-09T06:19:06,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146036 2024-12-09T06:19:06,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146036 2024-12-09T06:19:06,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146036) roll requested 2024-12-09T06:19:06,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146044 2024-12-09T06:19:06,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146044 2024-12-09T06:19:06,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146044) roll requested 2024-12-09T06:19:06,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146052 2024-12-09T06:19:06,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146052 2024-12-09T06:19:06,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146052) roll requested 2024-12-09T06:19:06,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146062 2024-12-09T06:19:06,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146062 2024-12-09T06:19:06,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146062) roll requested 2024-12-09T06:19:06,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146072 2024-12-09T06:19:06,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146072 2024-12-09T06:19:06,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146072) roll requested 2024-12-09T06:19:06,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146079 2024-12-09T06:19:06,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146079 2024-12-09T06:19:06,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146079) roll requested 2024-12-09T06:19:06,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146088 2024-12-09T06:19:06,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146088 2024-12-09T06:19:06,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146088) roll requested 2024-12-09T06:19:06,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146096 2024-12-09T06:19:06,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146096 2024-12-09T06:19:06,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146096) roll requested 2024-12-09T06:19:06,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146104 2024-12-09T06:19:06,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146104 2024-12-09T06:19:06,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146104) roll requested 2024-12-09T06:19:06,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146111 2024-12-09T06:19:06,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146111 2024-12-09T06:19:06,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146111) roll requested 2024-12-09T06:19:06,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146120 2024-12-09T06:19:06,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146120 2024-12-09T06:19:06,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146120) roll requested 2024-12-09T06:19:06,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146129 2024-12-09T06:19:06,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146129 2024-12-09T06:19:06,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146129) roll requested 2024-12-09T06:19:06,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146137 2024-12-09T06:19:06,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146137 2024-12-09T06:19:06,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146137) roll requested 2024-12-09T06:19:06,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146142 2024-12-09T06:19:06,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146142 2024-12-09T06:19:06,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146142) roll requested 2024-12-09T06:19:06,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146153 2024-12-09T06:19:06,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146153 2024-12-09T06:19:06,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146153) roll requested 2024-12-09T06:19:06,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146160 2024-12-09T06:19:06,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146160 2024-12-09T06:19:06,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146160) roll requested 2024-12-09T06:19:06,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146165 2024-12-09T06:19:06,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146165 2024-12-09T06:19:06,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146165) roll requested 2024-12-09T06:19:06,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146170 2024-12-09T06:19:06,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146170 2024-12-09T06:19:06,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146170) roll requested 2024-12-09T06:19:06,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146176 2024-12-09T06:19:06,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146176 2024-12-09T06:19:06,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146176) roll requested 2024-12-09T06:19:06,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146188 2024-12-09T06:19:06,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146188 2024-12-09T06:19:06,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146188) roll requested 2024-12-09T06:19:06,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146193 2024-12-09T06:19:06,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146193 2024-12-09T06:19:06,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146193) roll requested 2024-12-09T06:19:06,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146199 2024-12-09T06:19:06,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146199 2024-12-09T06:19:06,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146199) roll requested 2024-12-09T06:19:06,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146204 2024-12-09T06:19:06,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146204 2024-12-09T06:19:06,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146204) roll requested 2024-12-09T06:19:06,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146210 2024-12-09T06:19:06,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146210 2024-12-09T06:19:06,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146210) roll requested 2024-12-09T06:19:06,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146217 2024-12-09T06:19:06,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146217 2024-12-09T06:19:06,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146217) roll requested 2024-12-09T06:19:06,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146226 2024-12-09T06:19:06,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146226 2024-12-09T06:19:06,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146226) roll requested 2024-12-09T06:19:06,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146234 2024-12-09T06:19:06,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146234 2024-12-09T06:19:06,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146234) roll requested 2024-12-09T06:19:06,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146241 2024-12-09T06:19:06,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146241 2024-12-09T06:19:06,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146241) roll requested 2024-12-09T06:19:06,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146249 2024-12-09T06:19:06,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146249 2024-12-09T06:19:06,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146249) roll requested 2024-12-09T06:19:06,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146259 2024-12-09T06:19:06,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146259 2024-12-09T06:19:06,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146259) roll requested 2024-12-09T06:19:06,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146270 2024-12-09T06:19:06,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146270 2024-12-09T06:19:06,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146270) roll requested 2024-12-09T06:19:06,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146280 2024-12-09T06:19:06,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146280 2024-12-09T06:19:06,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146280) roll requested 2024-12-09T06:19:06,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146288 2024-12-09T06:19:06,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146288 2024-12-09T06:19:06,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146288) roll requested 2024-12-09T06:19:06,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146297 2024-12-09T06:19:06,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146297 2024-12-09T06:19:06,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146297) roll requested 2024-12-09T06:19:06,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146306 2024-12-09T06:19:06,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146306 2024-12-09T06:19:06,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146306) roll requested 2024-12-09T06:19:06,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146317 2024-12-09T06:19:06,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146317 2024-12-09T06:19:06,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146317) roll requested 2024-12-09T06:19:06,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146323 2024-12-09T06:19:06,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146323 2024-12-09T06:19:06,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146323) roll requested 2024-12-09T06:19:06,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146331 2024-12-09T06:19:06,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146331 2024-12-09T06:19:06,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146331) roll requested 2024-12-09T06:19:06,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146338 2024-12-09T06:19:06,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146338 2024-12-09T06:19:06,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146338) roll requested 2024-12-09T06:19:06,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146344 2024-12-09T06:19:06,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146344 2024-12-09T06:19:06,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146344) roll requested 2024-12-09T06:19:06,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146350 2024-12-09T06:19:06,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146350 2024-12-09T06:19:06,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146350) roll requested 2024-12-09T06:19:06,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146357 2024-12-09T06:19:06,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146357 2024-12-09T06:19:06,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146357) roll requested 2024-12-09T06:19:06,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146365 2024-12-09T06:19:06,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146365 2024-12-09T06:19:06,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146365) roll requested 2024-12-09T06:19:06,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146373 2024-12-09T06:19:06,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146373 2024-12-09T06:19:06,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146373) roll requested 2024-12-09T06:19:06,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146378 2024-12-09T06:19:06,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146378 2024-12-09T06:19:06,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146378) roll requested 2024-12-09T06:19:06,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146383 2024-12-09T06:19:06,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146383 2024-12-09T06:19:06,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146383) roll requested 2024-12-09T06:19:06,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146388 2024-12-09T06:19:06,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146388 2024-12-09T06:19:06,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146388) roll requested 2024-12-09T06:19:06,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146393 2024-12-09T06:19:06,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146393 2024-12-09T06:19:06,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146393) roll requested 2024-12-09T06:19:06,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146400 2024-12-09T06:19:06,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146400 2024-12-09T06:19:06,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146400) roll requested 2024-12-09T06:19:06,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146407 2024-12-09T06:19:06,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146407 2024-12-09T06:19:06,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146407) roll requested 2024-12-09T06:19:06,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146416 2024-12-09T06:19:06,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146416 2024-12-09T06:19:06,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146416) roll requested 2024-12-09T06:19:06,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146423 2024-12-09T06:19:06,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146423 2024-12-09T06:19:06,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146423) roll requested 2024-12-09T06:19:06,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146430 2024-12-09T06:19:06,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146430 2024-12-09T06:19:06,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146430) roll requested 2024-12-09T06:19:06,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146438 2024-12-09T06:19:06,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146438 2024-12-09T06:19:06,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146438) roll requested 2024-12-09T06:19:06,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146448 2024-12-09T06:19:06,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146448 2024-12-09T06:19:06,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146448) roll requested 2024-12-09T06:19:06,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146453 2024-12-09T06:19:06,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146453 2024-12-09T06:19:06,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146453) roll requested 2024-12-09T06:19:06,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146458 2024-12-09T06:19:06,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146458 2024-12-09T06:19:06,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146458) roll requested 2024-12-09T06:19:06,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146465 2024-12-09T06:19:06,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146465 2024-12-09T06:19:06,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146465) roll requested 2024-12-09T06:19:06,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146472 2024-12-09T06:19:06,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146472 2024-12-09T06:19:06,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146472) roll requested 2024-12-09T06:19:06,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146479 2024-12-09T06:19:06,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146479 2024-12-09T06:19:06,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146479) roll requested 2024-12-09T06:19:06,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146486 2024-12-09T06:19:06,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146486 2024-12-09T06:19:06,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146486) roll requested 2024-12-09T06:19:06,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146495 2024-12-09T06:19:06,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146495 2024-12-09T06:19:06,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146495) roll requested 2024-12-09T06:19:06,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146501 2024-12-09T06:19:06,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146501 2024-12-09T06:19:06,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146501) roll requested 2024-12-09T06:19:06,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146506 2024-12-09T06:19:06,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146506 2024-12-09T06:19:06,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146506) roll requested 2024-12-09T06:19:06,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146514 2024-12-09T06:19:06,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146514 2024-12-09T06:19:06,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146514) roll requested 2024-12-09T06:19:06,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146522 2024-12-09T06:19:06,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146522 2024-12-09T06:19:06,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146522) roll requested 2024-12-09T06:19:06,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146533 2024-12-09T06:19:06,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146533 2024-12-09T06:19:06,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146533) roll requested 2024-12-09T06:19:06,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146542 2024-12-09T06:19:06,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146542 2024-12-09T06:19:06,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146542) roll requested 2024-12-09T06:19:06,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146551 2024-12-09T06:19:06,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146551 2024-12-09T06:19:06,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146551) roll requested 2024-12-09T06:19:06,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146561 2024-12-09T06:19:06,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146561 2024-12-09T06:19:06,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146561) roll requested 2024-12-09T06:19:06,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146569 2024-12-09T06:19:06,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146569 2024-12-09T06:19:06,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146569) roll requested 2024-12-09T06:19:06,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146577 2024-12-09T06:19:06,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146577 2024-12-09T06:19:06,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146577) roll requested 2024-12-09T06:19:06,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146585 2024-12-09T06:19:06,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146585 2024-12-09T06:19:06,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146585) roll requested 2024-12-09T06:19:06,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146591 2024-12-09T06:19:06,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146591 2024-12-09T06:19:06,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146591) roll requested 2024-12-09T06:19:06,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146602 2024-12-09T06:19:06,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146602 2024-12-09T06:19:06,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146602) roll requested 2024-12-09T06:19:06,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146609 2024-12-09T06:19:06,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146609 2024-12-09T06:19:06,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146609) roll requested 2024-12-09T06:19:06,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146619 2024-12-09T06:19:06,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146619 2024-12-09T06:19:06,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146619) roll requested 2024-12-09T06:19:06,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146631 2024-12-09T06:19:06,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146631 2024-12-09T06:19:06,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146631) roll requested 2024-12-09T06:19:06,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146649 2024-12-09T06:19:06,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146649 2024-12-09T06:19:06,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146649) roll requested 2024-12-09T06:19:06,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146661 2024-12-09T06:19:06,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146661 2024-12-09T06:19:06,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146661) roll requested 2024-12-09T06:19:06,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146669 2024-12-09T06:19:06,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146669 2024-12-09T06:19:06,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146669) roll requested 2024-12-09T06:19:06,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146677 2024-12-09T06:19:06,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146677 2024-12-09T06:19:06,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146677) roll requested 2024-12-09T06:19:06,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146683 2024-12-09T06:19:06,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146683 2024-12-09T06:19:06,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146683) roll requested 2024-12-09T06:19:06,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146689 2024-12-09T06:19:06,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146689 2024-12-09T06:19:06,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146689) roll requested 2024-12-09T06:19:06,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146697 2024-12-09T06:19:06,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146697 2024-12-09T06:19:06,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146697) roll requested 2024-12-09T06:19:06,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146703 2024-12-09T06:19:06,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146703 2024-12-09T06:19:06,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146703) roll requested 2024-12-09T06:19:06,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146708 2024-12-09T06:19:06,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146708 2024-12-09T06:19:06,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146708) roll requested 2024-12-09T06:19:06,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146715 2024-12-09T06:19:06,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146715 2024-12-09T06:19:06,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146715) roll requested 2024-12-09T06:19:06,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146722 2024-12-09T06:19:06,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146722 2024-12-09T06:19:06,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146722) roll requested 2024-12-09T06:19:06,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146738 2024-12-09T06:19:06,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146738 2024-12-09T06:19:06,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146738) roll requested 2024-12-09T06:19:06,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146746 2024-12-09T06:19:06,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146746 2024-12-09T06:19:06,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146746) roll requested 2024-12-09T06:19:06,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146753 2024-12-09T06:19:06,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146753 2024-12-09T06:19:06,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146753) roll requested 2024-12-09T06:19:06,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146758 2024-12-09T06:19:06,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146758 2024-12-09T06:19:06,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146758) roll requested 2024-12-09T06:19:06,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146769 2024-12-09T06:19:06,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146769 2024-12-09T06:19:06,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146769) roll requested 2024-12-09T06:19:06,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146783 2024-12-09T06:19:06,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146783 2024-12-09T06:19:06,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146783) roll requested 2024-12-09T06:19:06,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146790 2024-12-09T06:19:06,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146790 2024-12-09T06:19:06,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146790) roll requested 2024-12-09T06:19:06,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146795 2024-12-09T06:19:06,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146795 2024-12-09T06:19:06,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146795) roll requested 2024-12-09T06:19:06,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146805 2024-12-09T06:19:06,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146805 2024-12-09T06:19:06,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146805) roll requested 2024-12-09T06:19:06,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146811 2024-12-09T06:19:06,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146811 2024-12-09T06:19:06,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146811) roll requested 2024-12-09T06:19:06,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146824 2024-12-09T06:19:06,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146824 2024-12-09T06:19:06,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146824) roll requested 2024-12-09T06:19:06,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146833 2024-12-09T06:19:06,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146833 2024-12-09T06:19:06,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146833) roll requested 2024-12-09T06:19:06,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146841 2024-12-09T06:19:06,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146841 2024-12-09T06:19:06,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146841) roll requested 2024-12-09T06:19:06,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146846 2024-12-09T06:19:06,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146846 2024-12-09T06:19:06,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146846) roll requested 2024-12-09T06:19:06,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146851 2024-12-09T06:19:06,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146851 2024-12-09T06:19:06,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146851) roll requested 2024-12-09T06:19:06,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146857 2024-12-09T06:19:06,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146857 2024-12-09T06:19:06,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146857) roll requested 2024-12-09T06:19:06,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146865 2024-12-09T06:19:06,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146865 2024-12-09T06:19:06,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146865) roll requested 2024-12-09T06:19:06,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146870 2024-12-09T06:19:06,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146870 2024-12-09T06:19:06,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146870) roll requested 2024-12-09T06:19:06,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146875 2024-12-09T06:19:06,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146875 2024-12-09T06:19:06,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146875) roll requested 2024-12-09T06:19:06,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146885 2024-12-09T06:19:06,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146885 2024-12-09T06:19:06,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146885) roll requested 2024-12-09T06:19:06,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146890 2024-12-09T06:19:06,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146890 2024-12-09T06:19:06,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146890) roll requested 2024-12-09T06:19:06,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146895 2024-12-09T06:19:06,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146895 2024-12-09T06:19:06,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146895) roll requested 2024-12-09T06:19:06,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146900 2024-12-09T06:19:06,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146900 2024-12-09T06:19:06,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146900) roll requested 2024-12-09T06:19:06,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146906 2024-12-09T06:19:06,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146906 2024-12-09T06:19:06,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146906) roll requested 2024-12-09T06:19:06,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146914 2024-12-09T06:19:06,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146914 2024-12-09T06:19:06,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146914) roll requested 2024-12-09T06:19:06,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146930 2024-12-09T06:19:06,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146930 2024-12-09T06:19:06,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146930) roll requested 2024-12-09T06:19:06,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146938 2024-12-09T06:19:06,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146938 2024-12-09T06:19:06,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146938) roll requested 2024-12-09T06:19:06,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146949 2024-12-09T06:19:06,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146949 2024-12-09T06:19:06,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146949) roll requested 2024-12-09T06:19:06,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146955 2024-12-09T06:19:06,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146955 2024-12-09T06:19:06,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146955) roll requested 2024-12-09T06:19:06,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146961 2024-12-09T06:19:06,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146961 2024-12-09T06:19:06,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146961) roll requested 2024-12-09T06:19:06,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146966 2024-12-09T06:19:06,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146966 2024-12-09T06:19:06,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146966) roll requested 2024-12-09T06:19:06,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146971 2024-12-09T06:19:06,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146971 2024-12-09T06:19:06,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146971) roll requested 2024-12-09T06:19:06,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146976 2024-12-09T06:19:06,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146976 2024-12-09T06:19:06,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146976) roll requested 2024-12-09T06:19:06,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146981 2024-12-09T06:19:06,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146981 2024-12-09T06:19:06,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146981) roll requested 2024-12-09T06:19:06,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146989 2024-12-09T06:19:06,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146989 2024-12-09T06:19:06,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:06,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:06,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146989) roll requested 2024-12-09T06:19:06,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725146995 2024-12-09T06:19:06,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:06,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146995 2024-12-09T06:19:07,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725146995) roll requested 2024-12-09T06:19:07,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147000 2024-12-09T06:19:07,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725146995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147000 2024-12-09T06:19:07,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147000) roll requested 2024-12-09T06:19:07,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147005 2024-12-09T06:19:07,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147005 2024-12-09T06:19:07,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147005) roll requested 2024-12-09T06:19:07,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147011 2024-12-09T06:19:07,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147011 2024-12-09T06:19:07,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147011) roll requested 2024-12-09T06:19:07,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147018 2024-12-09T06:19:07,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147018 2024-12-09T06:19:07,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147018) roll requested 2024-12-09T06:19:07,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147024 2024-12-09T06:19:07,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147024 2024-12-09T06:19:07,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147024) roll requested 2024-12-09T06:19:07,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147033 2024-12-09T06:19:07,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147033 2024-12-09T06:19:07,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147033) roll requested 2024-12-09T06:19:07,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147040 2024-12-09T06:19:07,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147040 2024-12-09T06:19:07,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147040) roll requested 2024-12-09T06:19:07,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147049 2024-12-09T06:19:07,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147049 2024-12-09T06:19:07,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147049) roll requested 2024-12-09T06:19:07,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147056 2024-12-09T06:19:07,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147056 2024-12-09T06:19:07,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147056) roll requested 2024-12-09T06:19:07,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147066 2024-12-09T06:19:07,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147066 2024-12-09T06:19:07,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147066) roll requested 2024-12-09T06:19:07,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147075 2024-12-09T06:19:07,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147075 2024-12-09T06:19:07,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147075) roll requested 2024-12-09T06:19:07,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147083 2024-12-09T06:19:07,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147083 2024-12-09T06:19:07,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147083) roll requested 2024-12-09T06:19:07,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147091 2024-12-09T06:19:07,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147091 2024-12-09T06:19:07,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147091) roll requested 2024-12-09T06:19:07,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147098 2024-12-09T06:19:07,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147098 2024-12-09T06:19:07,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147098) roll requested 2024-12-09T06:19:07,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147104 2024-12-09T06:19:07,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147104 2024-12-09T06:19:07,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147104) roll requested 2024-12-09T06:19:07,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147109 2024-12-09T06:19:07,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147109 2024-12-09T06:19:07,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147109) roll requested 2024-12-09T06:19:07,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147114 2024-12-09T06:19:07,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147114 2024-12-09T06:19:07,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147114) roll requested 2024-12-09T06:19:07,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147119 2024-12-09T06:19:07,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147119 2024-12-09T06:19:07,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147119) roll requested 2024-12-09T06:19:07,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147124 2024-12-09T06:19:07,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147124 2024-12-09T06:19:07,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147124) roll requested 2024-12-09T06:19:07,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147130 2024-12-09T06:19:07,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147130 2024-12-09T06:19:07,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147130) roll requested 2024-12-09T06:19:07,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147137 2024-12-09T06:19:07,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147137 2024-12-09T06:19:07,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147137) roll requested 2024-12-09T06:19:07,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147145 2024-12-09T06:19:07,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147145 2024-12-09T06:19:07,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147145) roll requested 2024-12-09T06:19:07,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147153 2024-12-09T06:19:07,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147153 2024-12-09T06:19:07,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147153) roll requested 2024-12-09T06:19:07,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147159 2024-12-09T06:19:07,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147159 2024-12-09T06:19:07,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147159) roll requested 2024-12-09T06:19:07,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147165 2024-12-09T06:19:07,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147165 2024-12-09T06:19:07,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147165) roll requested 2024-12-09T06:19:07,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147172 2024-12-09T06:19:07,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147172 2024-12-09T06:19:07,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147172) roll requested 2024-12-09T06:19:07,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147178 2024-12-09T06:19:07,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147178 2024-12-09T06:19:07,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147178) roll requested 2024-12-09T06:19:07,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147183 2024-12-09T06:19:07,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147183 2024-12-09T06:19:07,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147183) roll requested 2024-12-09T06:19:07,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147188 2024-12-09T06:19:07,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147188 2024-12-09T06:19:07,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147188) roll requested 2024-12-09T06:19:07,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147194 2024-12-09T06:19:07,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147194 2024-12-09T06:19:07,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147194) roll requested 2024-12-09T06:19:07,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147202 2024-12-09T06:19:07,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147202 2024-12-09T06:19:07,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147202) roll requested 2024-12-09T06:19:07,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147210 2024-12-09T06:19:07,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147210 2024-12-09T06:19:07,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147210) roll requested 2024-12-09T06:19:07,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147219 2024-12-09T06:19:07,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147219 2024-12-09T06:19:07,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147219) roll requested 2024-12-09T06:19:07,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147228 2024-12-09T06:19:07,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147228 2024-12-09T06:19:07,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147228) roll requested 2024-12-09T06:19:07,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147235 2024-12-09T06:19:07,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147235 2024-12-09T06:19:07,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147235) roll requested 2024-12-09T06:19:07,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147244 2024-12-09T06:19:07,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147244 2024-12-09T06:19:07,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147244) roll requested 2024-12-09T06:19:07,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147254 2024-12-09T06:19:07,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147254 2024-12-09T06:19:07,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147254) roll requested 2024-12-09T06:19:07,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147265 2024-12-09T06:19:07,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147265 2024-12-09T06:19:07,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147265) roll requested 2024-12-09T06:19:07,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147273 2024-12-09T06:19:07,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147273 2024-12-09T06:19:07,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147273) roll requested 2024-12-09T06:19:07,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147283 2024-12-09T06:19:07,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147283 2024-12-09T06:19:07,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147283) roll requested 2024-12-09T06:19:07,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147289 2024-12-09T06:19:07,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147289 2024-12-09T06:19:07,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147289) roll requested 2024-12-09T06:19:07,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147294 2024-12-09T06:19:07,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147294 2024-12-09T06:19:07,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147294) roll requested 2024-12-09T06:19:07,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147299 2024-12-09T06:19:07,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147299 2024-12-09T06:19:07,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147299) roll requested 2024-12-09T06:19:07,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147305 2024-12-09T06:19:07,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147305 2024-12-09T06:19:07,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147305) roll requested 2024-12-09T06:19:07,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147311 2024-12-09T06:19:07,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147311 2024-12-09T06:19:07,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147311) roll requested 2024-12-09T06:19:07,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147318 2024-12-09T06:19:07,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147318 2024-12-09T06:19:07,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147318) roll requested 2024-12-09T06:19:07,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147324 2024-12-09T06:19:07,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147324 2024-12-09T06:19:07,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147324) roll requested 2024-12-09T06:19:07,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147334 2024-12-09T06:19:07,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147334 2024-12-09T06:19:07,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147334) roll requested 2024-12-09T06:19:07,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147343 2024-12-09T06:19:07,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147343 2024-12-09T06:19:07,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147343) roll requested 2024-12-09T06:19:07,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147353 2024-12-09T06:19:07,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147353 2024-12-09T06:19:07,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147353) roll requested 2024-12-09T06:19:07,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147360 2024-12-09T06:19:07,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147360 2024-12-09T06:19:07,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147360) roll requested 2024-12-09T06:19:07,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147365 2024-12-09T06:19:07,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147365 2024-12-09T06:19:07,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147365) roll requested 2024-12-09T06:19:07,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147371 2024-12-09T06:19:07,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147371 2024-12-09T06:19:07,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147371) roll requested 2024-12-09T06:19:07,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147377 2024-12-09T06:19:07,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147377 2024-12-09T06:19:07,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147377) roll requested 2024-12-09T06:19:07,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147383 2024-12-09T06:19:07,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147383 2024-12-09T06:19:07,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147383) roll requested 2024-12-09T06:19:07,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147392 2024-12-09T06:19:07,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147392 2024-12-09T06:19:07,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147392) roll requested 2024-12-09T06:19:07,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147399 2024-12-09T06:19:07,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147399 2024-12-09T06:19:07,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147399) roll requested 2024-12-09T06:19:07,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147407 2024-12-09T06:19:07,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147407 2024-12-09T06:19:07,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147407) roll requested 2024-12-09T06:19:07,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147416 2024-12-09T06:19:07,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147416 2024-12-09T06:19:07,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147416) roll requested 2024-12-09T06:19:07,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147425 2024-12-09T06:19:07,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147425 2024-12-09T06:19:07,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147425) roll requested 2024-12-09T06:19:07,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147434 2024-12-09T06:19:07,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147434 2024-12-09T06:19:07,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147434) roll requested 2024-12-09T06:19:07,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147443 2024-12-09T06:19:07,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147443 2024-12-09T06:19:07,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147443) roll requested 2024-12-09T06:19:07,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147451 2024-12-09T06:19:07,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147451 2024-12-09T06:19:07,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147451) roll requested 2024-12-09T06:19:07,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147458 2024-12-09T06:19:07,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147458 2024-12-09T06:19:07,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147458) roll requested 2024-12-09T06:19:07,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147467 2024-12-09T06:19:07,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147467 2024-12-09T06:19:07,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147467) roll requested 2024-12-09T06:19:07,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147474 2024-12-09T06:19:07,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147474 2024-12-09T06:19:07,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147474) roll requested 2024-12-09T06:19:07,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147480 2024-12-09T06:19:07,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147480 2024-12-09T06:19:07,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147480) roll requested 2024-12-09T06:19:07,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147485 2024-12-09T06:19:07,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147485 2024-12-09T06:19:07,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147485) roll requested 2024-12-09T06:19:07,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147490 2024-12-09T06:19:07,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147490 2024-12-09T06:19:07,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147490) roll requested 2024-12-09T06:19:07,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147496 2024-12-09T06:19:07,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147496 2024-12-09T06:19:07,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147496) roll requested 2024-12-09T06:19:07,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147503 2024-12-09T06:19:07,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147503 2024-12-09T06:19:07,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147503) roll requested 2024-12-09T06:19:07,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147514 2024-12-09T06:19:07,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147514 2024-12-09T06:19:07,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147514) roll requested 2024-12-09T06:19:07,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147528 2024-12-09T06:19:07,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147528 2024-12-09T06:19:07,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147528) roll requested 2024-12-09T06:19:07,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147537 2024-12-09T06:19:07,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147537 2024-12-09T06:19:07,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147537) roll requested 2024-12-09T06:19:07,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147543 2024-12-09T06:19:07,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147543 2024-12-09T06:19:07,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147543) roll requested 2024-12-09T06:19:07,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147550 2024-12-09T06:19:07,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147550 2024-12-09T06:19:07,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147550) roll requested 2024-12-09T06:19:07,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147556 2024-12-09T06:19:07,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147556 2024-12-09T06:19:07,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147556) roll requested 2024-12-09T06:19:07,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147565 2024-12-09T06:19:07,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147565 2024-12-09T06:19:07,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147565) roll requested 2024-12-09T06:19:07,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147577 2024-12-09T06:19:07,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147577 2024-12-09T06:19:07,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147577) roll requested 2024-12-09T06:19:07,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147584 2024-12-09T06:19:07,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147584 2024-12-09T06:19:07,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147584) roll requested 2024-12-09T06:19:07,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147589 2024-12-09T06:19:07,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147589 2024-12-09T06:19:07,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147589) roll requested 2024-12-09T06:19:07,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147595 2024-12-09T06:19:07,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147595 2024-12-09T06:19:07,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147595) roll requested 2024-12-09T06:19:07,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147601 2024-12-09T06:19:07,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147601 2024-12-09T06:19:07,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147601) roll requested 2024-12-09T06:19:07,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147609 2024-12-09T06:19:07,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147609 2024-12-09T06:19:07,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147609) roll requested 2024-12-09T06:19:07,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147617 2024-12-09T06:19:07,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147617 2024-12-09T06:19:07,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147617) roll requested 2024-12-09T06:19:07,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147629 2024-12-09T06:19:07,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147629 2024-12-09T06:19:07,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147629) roll requested 2024-12-09T06:19:07,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147635 2024-12-09T06:19:07,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147635 2024-12-09T06:19:07,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147635) roll requested 2024-12-09T06:19:07,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147647 2024-12-09T06:19:07,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147647 2024-12-09T06:19:07,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147647) roll requested 2024-12-09T06:19:07,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147657 2024-12-09T06:19:07,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147657 2024-12-09T06:19:07,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147657) roll requested 2024-12-09T06:19:07,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147662 2024-12-09T06:19:07,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147662 2024-12-09T06:19:07,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147662) roll requested 2024-12-09T06:19:07,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147668 2024-12-09T06:19:07,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147668 2024-12-09T06:19:07,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147668) roll requested 2024-12-09T06:19:07,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147674 2024-12-09T06:19:07,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147674 2024-12-09T06:19:07,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147674) roll requested 2024-12-09T06:19:07,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147681 2024-12-09T06:19:07,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147681 2024-12-09T06:19:07,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147681) roll requested 2024-12-09T06:19:07,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147687 2024-12-09T06:19:07,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147687 2024-12-09T06:19:07,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147687) roll requested 2024-12-09T06:19:07,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147692 2024-12-09T06:19:07,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147692 2024-12-09T06:19:07,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147692) roll requested 2024-12-09T06:19:07,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147704 2024-12-09T06:19:07,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147704 2024-12-09T06:19:07,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147704) roll requested 2024-12-09T06:19:07,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147710 2024-12-09T06:19:07,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147710 2024-12-09T06:19:07,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147710) roll requested 2024-12-09T06:19:07,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147717 2024-12-09T06:19:07,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147717 2024-12-09T06:19:07,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147717) roll requested 2024-12-09T06:19:07,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147725 2024-12-09T06:19:07,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147725 2024-12-09T06:19:07,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147725) roll requested 2024-12-09T06:19:07,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147733 2024-12-09T06:19:07,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147733 2024-12-09T06:19:07,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147733) roll requested 2024-12-09T06:19:07,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147742 2024-12-09T06:19:07,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147742 2024-12-09T06:19:07,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147742) roll requested 2024-12-09T06:19:07,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147747 2024-12-09T06:19:07,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147747 2024-12-09T06:19:07,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147747) roll requested 2024-12-09T06:19:07,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147756 2024-12-09T06:19:07,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147756 2024-12-09T06:19:07,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147756) roll requested 2024-12-09T06:19:07,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147761 2024-12-09T06:19:07,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147761 2024-12-09T06:19:07,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147761) roll requested 2024-12-09T06:19:07,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147766 2024-12-09T06:19:07,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147766 2024-12-09T06:19:07,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147766) roll requested 2024-12-09T06:19:07,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147773 2024-12-09T06:19:07,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147773 2024-12-09T06:19:07,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147773) roll requested 2024-12-09T06:19:07,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147779 2024-12-09T06:19:07,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147779 2024-12-09T06:19:07,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147779) roll requested 2024-12-09T06:19:07,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147785 2024-12-09T06:19:07,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147785 2024-12-09T06:19:07,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147785) roll requested 2024-12-09T06:19:07,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147793 2024-12-09T06:19:07,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147793 2024-12-09T06:19:07,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147793) roll requested 2024-12-09T06:19:07,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147801 2024-12-09T06:19:07,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147801 2024-12-09T06:19:07,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147801) roll requested 2024-12-09T06:19:07,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147806 2024-12-09T06:19:07,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147806 2024-12-09T06:19:07,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147806) roll requested 2024-12-09T06:19:07,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147812 2024-12-09T06:19:07,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147812 2024-12-09T06:19:07,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147812) roll requested 2024-12-09T06:19:07,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147817 2024-12-09T06:19:07,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147817 2024-12-09T06:19:07,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147817) roll requested 2024-12-09T06:19:07,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147822 2024-12-09T06:19:07,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147822 2024-12-09T06:19:07,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147822) roll requested 2024-12-09T06:19:07,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147827 2024-12-09T06:19:07,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147827 2024-12-09T06:19:07,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147827) roll requested 2024-12-09T06:19:07,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147833 2024-12-09T06:19:07,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147833 2024-12-09T06:19:07,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147833) roll requested 2024-12-09T06:19:07,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147840 2024-12-09T06:19:07,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147840 2024-12-09T06:19:07,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147840) roll requested 2024-12-09T06:19:07,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147847 2024-12-09T06:19:07,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147847 2024-12-09T06:19:07,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147847) roll requested 2024-12-09T06:19:07,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147856 2024-12-09T06:19:07,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147856 2024-12-09T06:19:07,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147856) roll requested 2024-12-09T06:19:07,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147864 2024-12-09T06:19:07,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147864 2024-12-09T06:19:07,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147864) roll requested 2024-12-09T06:19:07,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147871 2024-12-09T06:19:07,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147871 2024-12-09T06:19:07,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147871) roll requested 2024-12-09T06:19:07,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147880 2024-12-09T06:19:07,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147880 2024-12-09T06:19:07,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147880) roll requested 2024-12-09T06:19:07,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147888 2024-12-09T06:19:07,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147888 2024-12-09T06:19:07,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147888) roll requested 2024-12-09T06:19:07,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147894 2024-12-09T06:19:07,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147894 2024-12-09T06:19:07,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147894) roll requested 2024-12-09T06:19:07,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147900 2024-12-09T06:19:07,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147900 2024-12-09T06:19:07,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147900) roll requested 2024-12-09T06:19:07,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147906 2024-12-09T06:19:07,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147906 2024-12-09T06:19:07,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147906) roll requested 2024-12-09T06:19:07,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147914 2024-12-09T06:19:07,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147914 2024-12-09T06:19:07,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147914) roll requested 2024-12-09T06:19:07,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147922 2024-12-09T06:19:07,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147922 2024-12-09T06:19:07,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147922) roll requested 2024-12-09T06:19:07,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147930 2024-12-09T06:19:07,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147930 2024-12-09T06:19:07,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147930) roll requested 2024-12-09T06:19:07,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147937 2024-12-09T06:19:07,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147937 2024-12-09T06:19:07,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147937) roll requested 2024-12-09T06:19:07,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147946 2024-12-09T06:19:07,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147946 2024-12-09T06:19:07,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147946) roll requested 2024-12-09T06:19:07,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147954 2024-12-09T06:19:07,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147954 2024-12-09T06:19:07,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147954) roll requested 2024-12-09T06:19:07,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147961 2024-12-09T06:19:07,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147961 2024-12-09T06:19:07,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147961) roll requested 2024-12-09T06:19:07,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147970 2024-12-09T06:19:07,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147970 2024-12-09T06:19:07,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147970) roll requested 2024-12-09T06:19:07,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147980 2024-12-09T06:19:07,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147980 2024-12-09T06:19:07,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147980) roll requested 2024-12-09T06:19:07,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147988 2024-12-09T06:19:07,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:07,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147988 2024-12-09T06:19:07,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:07,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:07,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147988) roll requested 2024-12-09T06:19:07,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725147994 2024-12-09T06:19:08,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147994 2024-12-09T06:19:08,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725147994) roll requested 2024-12-09T06:19:08,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148002 2024-12-09T06:19:08,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725147994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148002 2024-12-09T06:19:08,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148002) roll requested 2024-12-09T06:19:08,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148010 2024-12-09T06:19:08,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148010 2024-12-09T06:19:08,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148010) roll requested 2024-12-09T06:19:08,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148020 2024-12-09T06:19:08,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148020 2024-12-09T06:19:08,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148020) roll requested 2024-12-09T06:19:08,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148032 2024-12-09T06:19:08,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148032 2024-12-09T06:19:08,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148032) roll requested 2024-12-09T06:19:08,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148041 2024-12-09T06:19:08,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148041 2024-12-09T06:19:08,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148041) roll requested 2024-12-09T06:19:08,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148046 2024-12-09T06:19:08,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148046 2024-12-09T06:19:08,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148046) roll requested 2024-12-09T06:19:08,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148051 2024-12-09T06:19:08,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148051 2024-12-09T06:19:08,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148051) roll requested 2024-12-09T06:19:08,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148057 2024-12-09T06:19:08,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148057 2024-12-09T06:19:08,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148057) roll requested 2024-12-09T06:19:08,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148064 2024-12-09T06:19:08,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148064 2024-12-09T06:19:08,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148064) roll requested 2024-12-09T06:19:08,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148070 2024-12-09T06:19:08,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148070 2024-12-09T06:19:08,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148070) roll requested 2024-12-09T06:19:08,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148077 2024-12-09T06:19:08,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148077 2024-12-09T06:19:08,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148077) roll requested 2024-12-09T06:19:08,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148086 2024-12-09T06:19:08,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148086 2024-12-09T06:19:08,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148077 is not closed yet, will try archiving it next time 2024-12-09T06:19:08,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148086) roll requested 2024-12-09T06:19:08,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148094 2024-12-09T06:19:08,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148094 2024-12-09T06:19:08,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148094) roll requested 2024-12-09T06:19:08,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148102 2024-12-09T06:19:08,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148102 2024-12-09T06:19:08,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148102) roll requested 2024-12-09T06:19:08,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148111 2024-12-09T06:19:08,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148111 2024-12-09T06:19:08,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148111) roll requested 2024-12-09T06:19:08,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148118 2024-12-09T06:19:08,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148118 2024-12-09T06:19:08,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148118) roll requested 2024-12-09T06:19:08,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148124 2024-12-09T06:19:08,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148124 2024-12-09T06:19:08,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148124) roll requested 2024-12-09T06:19:08,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148129 2024-12-09T06:19:08,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148129 2024-12-09T06:19:08,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148129) roll requested 2024-12-09T06:19:08,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148136 2024-12-09T06:19:08,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148136 2024-12-09T06:19:08,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148136) roll requested 2024-12-09T06:19:08,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148145 2024-12-09T06:19:08,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148145 2024-12-09T06:19:08,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148145) roll requested 2024-12-09T06:19:08,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148150 2024-12-09T06:19:08,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148150 2024-12-09T06:19:08,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148150) roll requested 2024-12-09T06:19:08,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148155 2024-12-09T06:19:08,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148155 2024-12-09T06:19:08,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148155) roll requested 2024-12-09T06:19:08,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148160 2024-12-09T06:19:08,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148160 2024-12-09T06:19:08,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148160) roll requested 2024-12-09T06:19:08,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148166 2024-12-09T06:19:08,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148166 2024-12-09T06:19:08,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148166) roll requested 2024-12-09T06:19:08,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148173 2024-12-09T06:19:08,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148173 2024-12-09T06:19:08,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148173) roll requested 2024-12-09T06:19:08,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148181 2024-12-09T06:19:08,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148181 2024-12-09T06:19:08,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148181) roll requested 2024-12-09T06:19:08,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148188 2024-12-09T06:19:08,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148188 2024-12-09T06:19:08,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148188) roll requested 2024-12-09T06:19:08,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148195 2024-12-09T06:19:08,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148195 2024-12-09T06:19:08,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148195) roll requested 2024-12-09T06:19:08,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148202 2024-12-09T06:19:08,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148202 2024-12-09T06:19:08,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148202) roll requested 2024-12-09T06:19:08,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148211 2024-12-09T06:19:08,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148211 2024-12-09T06:19:08,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148211) roll requested 2024-12-09T06:19:08,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148220 2024-12-09T06:19:08,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148220 2024-12-09T06:19:08,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148220) roll requested 2024-12-09T06:19:08,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148227 2024-12-09T06:19:08,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148227 2024-12-09T06:19:08,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148227) roll requested 2024-12-09T06:19:08,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148234 2024-12-09T06:19:08,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148234 2024-12-09T06:19:08,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148234) roll requested 2024-12-09T06:19:08,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148242 2024-12-09T06:19:08,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148242 2024-12-09T06:19:08,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148242) roll requested 2024-12-09T06:19:08,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148250 2024-12-09T06:19:08,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148250 2024-12-09T06:19:08,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148250) roll requested 2024-12-09T06:19:08,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148259 2024-12-09T06:19:08,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148259 2024-12-09T06:19:08,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148259) roll requested 2024-12-09T06:19:08,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148268 2024-12-09T06:19:08,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148268 2024-12-09T06:19:08,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148268) roll requested 2024-12-09T06:19:08,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148274 2024-12-09T06:19:08,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148274 2024-12-09T06:19:08,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148274) roll requested 2024-12-09T06:19:08,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148283 2024-12-09T06:19:08,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148283 2024-12-09T06:19:08,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148283) roll requested 2024-12-09T06:19:08,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148291 2024-12-09T06:19:08,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148291 2024-12-09T06:19:08,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148291) roll requested 2024-12-09T06:19:08,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148297 2024-12-09T06:19:08,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148297 2024-12-09T06:19:08,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148297) roll requested 2024-12-09T06:19:08,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148305 2024-12-09T06:19:08,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148305 2024-12-09T06:19:08,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148305) roll requested 2024-12-09T06:19:08,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148312 2024-12-09T06:19:08,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148312 2024-12-09T06:19:08,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148312) roll requested 2024-12-09T06:19:08,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148320 2024-12-09T06:19:08,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148320 2024-12-09T06:19:08,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148320) roll requested 2024-12-09T06:19:08,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148328 2024-12-09T06:19:08,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148328 2024-12-09T06:19:08,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148328) roll requested 2024-12-09T06:19:08,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148335 2024-12-09T06:19:08,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148335 2024-12-09T06:19:08,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148335) roll requested 2024-12-09T06:19:08,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148341 2024-12-09T06:19:08,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148341 2024-12-09T06:19:08,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148341) roll requested 2024-12-09T06:19:08,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148348 2024-12-09T06:19:08,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148348 2024-12-09T06:19:08,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148348) roll requested 2024-12-09T06:19:08,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148357 2024-12-09T06:19:08,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148357 2024-12-09T06:19:08,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148357) roll requested 2024-12-09T06:19:08,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148365 2024-12-09T06:19:08,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148365 2024-12-09T06:19:08,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148365) roll requested 2024-12-09T06:19:08,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148370 2024-12-09T06:19:08,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148370 2024-12-09T06:19:08,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148370) roll requested 2024-12-09T06:19:08,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148376 2024-12-09T06:19:08,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148376 2024-12-09T06:19:08,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148376) roll requested 2024-12-09T06:19:08,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148383 2024-12-09T06:19:08,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148383 2024-12-09T06:19:08,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148383) roll requested 2024-12-09T06:19:08,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148390 2024-12-09T06:19:08,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148390 2024-12-09T06:19:08,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148390) roll requested 2024-12-09T06:19:08,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148397 2024-12-09T06:19:08,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148397 2024-12-09T06:19:08,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148397) roll requested 2024-12-09T06:19:08,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148404 2024-12-09T06:19:08,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148404 2024-12-09T06:19:08,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148404) roll requested 2024-12-09T06:19:08,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148412 2024-12-09T06:19:08,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148412 2024-12-09T06:19:08,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148412) roll requested 2024-12-09T06:19:08,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148419 2024-12-09T06:19:08,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148419 2024-12-09T06:19:08,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148419) roll requested 2024-12-09T06:19:08,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148428 2024-12-09T06:19:08,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148428 2024-12-09T06:19:08,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148428) roll requested 2024-12-09T06:19:08,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148437 2024-12-09T06:19:08,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148437 2024-12-09T06:19:08,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148437) roll requested 2024-12-09T06:19:08,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148446 2024-12-09T06:19:08,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148446 2024-12-09T06:19:08,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148446) roll requested 2024-12-09T06:19:08,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148452 2024-12-09T06:19:08,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148452 2024-12-09T06:19:08,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148452) roll requested 2024-12-09T06:19:08,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148459 2024-12-09T06:19:08,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148459 2024-12-09T06:19:08,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148459) roll requested 2024-12-09T06:19:08,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148466 2024-12-09T06:19:08,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148466 2024-12-09T06:19:08,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148466) roll requested 2024-12-09T06:19:08,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148475 2024-12-09T06:19:08,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148475 2024-12-09T06:19:08,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148475) roll requested 2024-12-09T06:19:08,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148484 2024-12-09T06:19:08,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148484 2024-12-09T06:19:08,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148484) roll requested 2024-12-09T06:19:08,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148496 2024-12-09T06:19:08,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148496 2024-12-09T06:19:08,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148496) roll requested 2024-12-09T06:19:08,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148503 2024-12-09T06:19:08,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148503 2024-12-09T06:19:08,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148503) roll requested 2024-12-09T06:19:08,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148512 2024-12-09T06:19:08,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148512 2024-12-09T06:19:08,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148512) roll requested 2024-12-09T06:19:08,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148519 2024-12-09T06:19:08,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148519 2024-12-09T06:19:08,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148519) roll requested 2024-12-09T06:19:08,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148525 2024-12-09T06:19:08,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148525 2024-12-09T06:19:08,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148525) roll requested 2024-12-09T06:19:08,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148531 2024-12-09T06:19:08,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148531 2024-12-09T06:19:08,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148531) roll requested 2024-12-09T06:19:08,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148536 2024-12-09T06:19:08,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148536 2024-12-09T06:19:08,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148536) roll requested 2024-12-09T06:19:08,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148543 2024-12-09T06:19:08,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148543 2024-12-09T06:19:08,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148543) roll requested 2024-12-09T06:19:08,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148551 2024-12-09T06:19:08,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148551 2024-12-09T06:19:08,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148551) roll requested 2024-12-09T06:19:08,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148558 2024-12-09T06:19:08,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148558 2024-12-09T06:19:08,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148558) roll requested 2024-12-09T06:19:08,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148567 2024-12-09T06:19:08,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148567 2024-12-09T06:19:08,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148567) roll requested 2024-12-09T06:19:08,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148576 2024-12-09T06:19:08,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148576 2024-12-09T06:19:08,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148576) roll requested 2024-12-09T06:19:08,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148585 2024-12-09T06:19:08,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148585 2024-12-09T06:19:08,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148585) roll requested 2024-12-09T06:19:08,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148595 2024-12-09T06:19:08,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148595 2024-12-09T06:19:08,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148595) roll requested 2024-12-09T06:19:08,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148605 2024-12-09T06:19:08,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148605 2024-12-09T06:19:08,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148605) roll requested 2024-12-09T06:19:08,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148612 2024-12-09T06:19:08,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148612 2024-12-09T06:19:08,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148612) roll requested 2024-12-09T06:19:08,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148619 2024-12-09T06:19:08,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148619 2024-12-09T06:19:08,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148619) roll requested 2024-12-09T06:19:08,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148625 2024-12-09T06:19:08,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148625 2024-12-09T06:19:08,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148625) roll requested 2024-12-09T06:19:08,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148632 2024-12-09T06:19:08,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148632 2024-12-09T06:19:08,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148632) roll requested 2024-12-09T06:19:08,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148639 2024-12-09T06:19:08,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148639 2024-12-09T06:19:08,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148639) roll requested 2024-12-09T06:19:08,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148645 2024-12-09T06:19:08,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148645 2024-12-09T06:19:08,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148645) roll requested 2024-12-09T06:19:08,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148651 2024-12-09T06:19:08,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148651 2024-12-09T06:19:08,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148651) roll requested 2024-12-09T06:19:08,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148657 2024-12-09T06:19:08,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148657 2024-12-09T06:19:08,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148657) roll requested 2024-12-09T06:19:08,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148665 2024-12-09T06:19:08,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148665 2024-12-09T06:19:08,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148665) roll requested 2024-12-09T06:19:08,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148671 2024-12-09T06:19:08,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148671 2024-12-09T06:19:08,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148671) roll requested 2024-12-09T06:19:08,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148677 2024-12-09T06:19:08,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148677 2024-12-09T06:19:08,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148677) roll requested 2024-12-09T06:19:08,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148697 2024-12-09T06:19:08,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148697 2024-12-09T06:19:08,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148697) roll requested 2024-12-09T06:19:08,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148713 2024-12-09T06:19:08,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148713 2024-12-09T06:19:08,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148713) roll requested 2024-12-09T06:19:08,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148730 2024-12-09T06:19:08,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148730 2024-12-09T06:19:08,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148730) roll requested 2024-12-09T06:19:08,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148745 2024-12-09T06:19:08,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148745 2024-12-09T06:19:08,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148745) roll requested 2024-12-09T06:19:08,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148754 2024-12-09T06:19:08,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148754 2024-12-09T06:19:08,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148754) roll requested 2024-12-09T06:19:08,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148761 2024-12-09T06:19:08,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148761 2024-12-09T06:19:08,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148761) roll requested 2024-12-09T06:19:08,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148771 2024-12-09T06:19:08,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148771 2024-12-09T06:19:08,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148771) roll requested 2024-12-09T06:19:08,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148781 2024-12-09T06:19:08,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148781 2024-12-09T06:19:08,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148781) roll requested 2024-12-09T06:19:08,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148789 2024-12-09T06:19:08,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148789 2024-12-09T06:19:08,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148789) roll requested 2024-12-09T06:19:08,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148795 2024-12-09T06:19:08,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148795 2024-12-09T06:19:08,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148795) roll requested 2024-12-09T06:19:08,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148800 2024-12-09T06:19:08,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148800 2024-12-09T06:19:08,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148800) roll requested 2024-12-09T06:19:08,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148806 2024-12-09T06:19:08,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148806 2024-12-09T06:19:08,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148806) roll requested 2024-12-09T06:19:08,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148812 2024-12-09T06:19:08,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148812 2024-12-09T06:19:08,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148812) roll requested 2024-12-09T06:19:08,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148818 2024-12-09T06:19:08,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148818 2024-12-09T06:19:08,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148818) roll requested 2024-12-09T06:19:08,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148824 2024-12-09T06:19:08,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148824 2024-12-09T06:19:08,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148824) roll requested 2024-12-09T06:19:08,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148831 2024-12-09T06:19:08,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148831 2024-12-09T06:19:08,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148831) roll requested 2024-12-09T06:19:08,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148840 2024-12-09T06:19:08,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148840 2024-12-09T06:19:08,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148840) roll requested 2024-12-09T06:19:08,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148848 2024-12-09T06:19:08,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148848 2024-12-09T06:19:08,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148848) roll requested 2024-12-09T06:19:08,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148856 2024-12-09T06:19:08,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148856 2024-12-09T06:19:08,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148856) roll requested 2024-12-09T06:19:08,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148865 2024-12-09T06:19:08,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148865 2024-12-09T06:19:08,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148865) roll requested 2024-12-09T06:19:08,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148871 2024-12-09T06:19:08,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148871 2024-12-09T06:19:08,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148871) roll requested 2024-12-09T06:19:08,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148879 2024-12-09T06:19:08,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148879 2024-12-09T06:19:08,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148879) roll requested 2024-12-09T06:19:08,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148885 2024-12-09T06:19:08,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148885 2024-12-09T06:19:08,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148885) roll requested 2024-12-09T06:19:08,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148893 2024-12-09T06:19:08,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148893 2024-12-09T06:19:08,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148893) roll requested 2024-12-09T06:19:08,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148901 2024-12-09T06:19:08,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148901 2024-12-09T06:19:08,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148901) roll requested 2024-12-09T06:19:08,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148910 2024-12-09T06:19:08,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148910 2024-12-09T06:19:08,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148910) roll requested 2024-12-09T06:19:08,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148919 2024-12-09T06:19:08,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148919 2024-12-09T06:19:08,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148919) roll requested 2024-12-09T06:19:08,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148929 2024-12-09T06:19:08,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148929 2024-12-09T06:19:08,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148929) roll requested 2024-12-09T06:19:08,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148934 2024-12-09T06:19:08,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148934 2024-12-09T06:19:08,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148934) roll requested 2024-12-09T06:19:08,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148939 2024-12-09T06:19:08,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148939 2024-12-09T06:19:08,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148939) roll requested 2024-12-09T06:19:08,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148945 2024-12-09T06:19:08,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148945 2024-12-09T06:19:08,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148945) roll requested 2024-12-09T06:19:08,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148952 2024-12-09T06:19:08,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148952 2024-12-09T06:19:08,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148952) roll requested 2024-12-09T06:19:08,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148959 2024-12-09T06:19:08,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148959 2024-12-09T06:19:08,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148959) roll requested 2024-12-09T06:19:08,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148968 2024-12-09T06:19:08,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148968 2024-12-09T06:19:08,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148968) roll requested 2024-12-09T06:19:08,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148976 2024-12-09T06:19:08,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148976 2024-12-09T06:19:08,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148976) roll requested 2024-12-09T06:19:08,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148984 2024-12-09T06:19:08,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148984 2024-12-09T06:19:08,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:08,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:08,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148984) roll requested 2024-12-09T06:19:08,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725148994 2024-12-09T06:19:08,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:08,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148994 2024-12-09T06:19:09,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725148994) roll requested 2024-12-09T06:19:09,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149001 2024-12-09T06:19:09,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725148994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149001 2024-12-09T06:19:09,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149001) roll requested 2024-12-09T06:19:09,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149006 2024-12-09T06:19:09,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149006 2024-12-09T06:19:09,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149006) roll requested 2024-12-09T06:19:09,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149014 2024-12-09T06:19:09,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149014 2024-12-09T06:19:09,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149014) roll requested 2024-12-09T06:19:09,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149021 2024-12-09T06:19:09,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149021 2024-12-09T06:19:09,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149021) roll requested 2024-12-09T06:19:09,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149028 2024-12-09T06:19:09,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149028 2024-12-09T06:19:09,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149028) roll requested 2024-12-09T06:19:09,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149033 2024-12-09T06:19:09,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149033 2024-12-09T06:19:09,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149033) roll requested 2024-12-09T06:19:09,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149041 2024-12-09T06:19:09,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149041 2024-12-09T06:19:09,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149041) roll requested 2024-12-09T06:19:09,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149046 2024-12-09T06:19:09,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149046 2024-12-09T06:19:09,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149046) roll requested 2024-12-09T06:19:09,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149052 2024-12-09T06:19:09,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149052 2024-12-09T06:19:09,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149052) roll requested 2024-12-09T06:19:09,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149060 2024-12-09T06:19:09,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149060 2024-12-09T06:19:09,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149060) roll requested 2024-12-09T06:19:09,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149067 2024-12-09T06:19:09,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149067 2024-12-09T06:19:09,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149067) roll requested 2024-12-09T06:19:09,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149085 2024-12-09T06:19:09,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149085 2024-12-09T06:19:09,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149085) roll requested 2024-12-09T06:19:09,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149093 2024-12-09T06:19:09,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149093 2024-12-09T06:19:09,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149093) roll requested 2024-12-09T06:19:09,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149100 2024-12-09T06:19:09,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149100 2024-12-09T06:19:09,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149100) roll requested 2024-12-09T06:19:09,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149108 2024-12-09T06:19:09,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149108 2024-12-09T06:19:09,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149108) roll requested 2024-12-09T06:19:09,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149117 2024-12-09T06:19:09,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149117 2024-12-09T06:19:09,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149117) roll requested 2024-12-09T06:19:09,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149125 2024-12-09T06:19:09,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149125 2024-12-09T06:19:09,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149125) roll requested 2024-12-09T06:19:09,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149131 2024-12-09T06:19:09,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149131 2024-12-09T06:19:09,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149131) roll requested 2024-12-09T06:19:09,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149136 2024-12-09T06:19:09,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149136 2024-12-09T06:19:09,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149136) roll requested 2024-12-09T06:19:09,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149142 2024-12-09T06:19:09,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149142 2024-12-09T06:19:09,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149142) roll requested 2024-12-09T06:19:09,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149148 2024-12-09T06:19:09,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149148 2024-12-09T06:19:09,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149148) roll requested 2024-12-09T06:19:09,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149153 2024-12-09T06:19:09,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149153 2024-12-09T06:19:09,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149153) roll requested 2024-12-09T06:19:09,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149159 2024-12-09T06:19:09,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149159 2024-12-09T06:19:09,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149159) roll requested 2024-12-09T06:19:09,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149167 2024-12-09T06:19:09,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149167 2024-12-09T06:19:09,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149167) roll requested 2024-12-09T06:19:09,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149173 2024-12-09T06:19:09,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149173 2024-12-09T06:19:09,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149173) roll requested 2024-12-09T06:19:09,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149180 2024-12-09T06:19:09,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149180 2024-12-09T06:19:09,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149180) roll requested 2024-12-09T06:19:09,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149188 2024-12-09T06:19:09,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149188 2024-12-09T06:19:09,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149188) roll requested 2024-12-09T06:19:09,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149198 2024-12-09T06:19:09,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149198 2024-12-09T06:19:09,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149198) roll requested 2024-12-09T06:19:09,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149205 2024-12-09T06:19:09,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149205 2024-12-09T06:19:09,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149205) roll requested 2024-12-09T06:19:09,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149213 2024-12-09T06:19:09,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149213 2024-12-09T06:19:09,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149213) roll requested 2024-12-09T06:19:09,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149218 2024-12-09T06:19:09,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149218 2024-12-09T06:19:09,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149218) roll requested 2024-12-09T06:19:09,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149227 2024-12-09T06:19:09,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149227 2024-12-09T06:19:09,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149227) roll requested 2024-12-09T06:19:09,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149232 2024-12-09T06:19:09,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149232 2024-12-09T06:19:09,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149232) roll requested 2024-12-09T06:19:09,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149237 2024-12-09T06:19:09,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149237 2024-12-09T06:19:09,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149237) roll requested 2024-12-09T06:19:09,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149242 2024-12-09T06:19:09,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149242 2024-12-09T06:19:09,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149242) roll requested 2024-12-09T06:19:09,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149247 2024-12-09T06:19:09,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149247 2024-12-09T06:19:09,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149247) roll requested 2024-12-09T06:19:09,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149253 2024-12-09T06:19:09,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149253 2024-12-09T06:19:09,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149253) roll requested 2024-12-09T06:19:09,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149259 2024-12-09T06:19:09,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149259 2024-12-09T06:19:09,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149259) roll requested 2024-12-09T06:19:09,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149266 2024-12-09T06:19:09,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149266 2024-12-09T06:19:09,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149266) roll requested 2024-12-09T06:19:09,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149275 2024-12-09T06:19:09,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149275 2024-12-09T06:19:09,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149275) roll requested 2024-12-09T06:19:09,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149284 2024-12-09T06:19:09,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149284 2024-12-09T06:19:09,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149284) roll requested 2024-12-09T06:19:09,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149290 2024-12-09T06:19:09,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149290 2024-12-09T06:19:09,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149290) roll requested 2024-12-09T06:19:09,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149295 2024-12-09T06:19:09,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149295 2024-12-09T06:19:09,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149295) roll requested 2024-12-09T06:19:09,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149301 2024-12-09T06:19:09,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149301 2024-12-09T06:19:09,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149301) roll requested 2024-12-09T06:19:09,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149306 2024-12-09T06:19:09,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149306 2024-12-09T06:19:09,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149306) roll requested 2024-12-09T06:19:09,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149316 2024-12-09T06:19:09,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149316 2024-12-09T06:19:09,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149316) roll requested 2024-12-09T06:19:09,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149323 2024-12-09T06:19:09,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149323 2024-12-09T06:19:09,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149323) roll requested 2024-12-09T06:19:09,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149331 2024-12-09T06:19:09,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149331 2024-12-09T06:19:09,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149331) roll requested 2024-12-09T06:19:09,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149340 2024-12-09T06:19:09,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149340 2024-12-09T06:19:09,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149340) roll requested 2024-12-09T06:19:09,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149346 2024-12-09T06:19:09,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149346 2024-12-09T06:19:09,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149346) roll requested 2024-12-09T06:19:09,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149350 2024-12-09T06:19:09,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149350 2024-12-09T06:19:09,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149350) roll requested 2024-12-09T06:19:09,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149355 2024-12-09T06:19:09,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149355 2024-12-09T06:19:09,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149355) roll requested 2024-12-09T06:19:09,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149360 2024-12-09T06:19:09,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149360 2024-12-09T06:19:09,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149360) roll requested 2024-12-09T06:19:09,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149365 2024-12-09T06:19:09,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149365 2024-12-09T06:19:09,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149365) roll requested 2024-12-09T06:19:09,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149372 2024-12-09T06:19:09,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149372 2024-12-09T06:19:09,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149372) roll requested 2024-12-09T06:19:09,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149379 2024-12-09T06:19:09,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149379 2024-12-09T06:19:09,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149379) roll requested 2024-12-09T06:19:09,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149386 2024-12-09T06:19:09,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149386 2024-12-09T06:19:09,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149386) roll requested 2024-12-09T06:19:09,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149396 2024-12-09T06:19:09,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149396 2024-12-09T06:19:09,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149396) roll requested 2024-12-09T06:19:09,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149403 2024-12-09T06:19:09,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149403 2024-12-09T06:19:09,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149403) roll requested 2024-12-09T06:19:09,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149412 2024-12-09T06:19:09,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149412 2024-12-09T06:19:09,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149412) roll requested 2024-12-09T06:19:09,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149421 2024-12-09T06:19:09,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149421 2024-12-09T06:19:09,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149421) roll requested 2024-12-09T06:19:09,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149427 2024-12-09T06:19:09,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149427 2024-12-09T06:19:09,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149427) roll requested 2024-12-09T06:19:09,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149433 2024-12-09T06:19:09,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149433 2024-12-09T06:19:09,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149433) roll requested 2024-12-09T06:19:09,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149439 2024-12-09T06:19:09,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149439 2024-12-09T06:19:09,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149439) roll requested 2024-12-09T06:19:09,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149445 2024-12-09T06:19:09,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149445 2024-12-09T06:19:09,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149445) roll requested 2024-12-09T06:19:09,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149451 2024-12-09T06:19:09,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149451 2024-12-09T06:19:09,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149451) roll requested 2024-12-09T06:19:09,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149461 2024-12-09T06:19:09,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149461 2024-12-09T06:19:09,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149461) roll requested 2024-12-09T06:19:09,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149467 2024-12-09T06:19:09,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149467 2024-12-09T06:19:09,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149467) roll requested 2024-12-09T06:19:09,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149474 2024-12-09T06:19:09,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149474 2024-12-09T06:19:09,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149474) roll requested 2024-12-09T06:19:09,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149480 2024-12-09T06:19:09,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149480 2024-12-09T06:19:09,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149480) roll requested 2024-12-09T06:19:09,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149485 2024-12-09T06:19:09,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149485 2024-12-09T06:19:09,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149485) roll requested 2024-12-09T06:19:09,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149491 2024-12-09T06:19:09,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149491 2024-12-09T06:19:09,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149491) roll requested 2024-12-09T06:19:09,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149498 2024-12-09T06:19:09,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149498 2024-12-09T06:19:09,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149498) roll requested 2024-12-09T06:19:09,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149506 2024-12-09T06:19:09,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149506 2024-12-09T06:19:09,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149506) roll requested 2024-12-09T06:19:09,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149516 2024-12-09T06:19:09,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149516 2024-12-09T06:19:09,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149516) roll requested 2024-12-09T06:19:09,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149524 2024-12-09T06:19:09,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149524 2024-12-09T06:19:09,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149524) roll requested 2024-12-09T06:19:09,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149530 2024-12-09T06:19:09,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149530 2024-12-09T06:19:09,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149530) roll requested 2024-12-09T06:19:09,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149536 2024-12-09T06:19:09,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149536 2024-12-09T06:19:09,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149536) roll requested 2024-12-09T06:19:09,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149542 2024-12-09T06:19:09,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149542 2024-12-09T06:19:09,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149542) roll requested 2024-12-09T06:19:09,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149548 2024-12-09T06:19:09,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149548 2024-12-09T06:19:09,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149548) roll requested 2024-12-09T06:19:09,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149553 2024-12-09T06:19:09,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149553 2024-12-09T06:19:09,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149553) roll requested 2024-12-09T06:19:09,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149557 2024-12-09T06:19:09,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149557 2024-12-09T06:19:09,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149557) roll requested 2024-12-09T06:19:09,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149563 2024-12-09T06:19:09,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149563 2024-12-09T06:19:09,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149563) roll requested 2024-12-09T06:19:09,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149569 2024-12-09T06:19:09,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149569 2024-12-09T06:19:09,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149569) roll requested 2024-12-09T06:19:09,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149576 2024-12-09T06:19:09,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149576 2024-12-09T06:19:09,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149576) roll requested 2024-12-09T06:19:09,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149582 2024-12-09T06:19:09,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149582 2024-12-09T06:19:09,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149582) roll requested 2024-12-09T06:19:09,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149587 2024-12-09T06:19:09,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149587 2024-12-09T06:19:09,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149587) roll requested 2024-12-09T06:19:09,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149594 2024-12-09T06:19:09,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149594 2024-12-09T06:19:09,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149594) roll requested 2024-12-09T06:19:09,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149601 2024-12-09T06:19:09,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149601 2024-12-09T06:19:09,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149601) roll requested 2024-12-09T06:19:09,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149609 2024-12-09T06:19:09,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149609 2024-12-09T06:19:09,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149609) roll requested 2024-12-09T06:19:09,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149618 2024-12-09T06:19:09,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149618 2024-12-09T06:19:09,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149618) roll requested 2024-12-09T06:19:09,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149627 2024-12-09T06:19:09,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149627 2024-12-09T06:19:09,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149627) roll requested 2024-12-09T06:19:09,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149635 2024-12-09T06:19:09,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149635 2024-12-09T06:19:09,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149635) roll requested 2024-12-09T06:19:09,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149644 2024-12-09T06:19:09,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149644 2024-12-09T06:19:09,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149644) roll requested 2024-12-09T06:19:09,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149653 2024-12-09T06:19:09,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149653 2024-12-09T06:19:09,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149653) roll requested 2024-12-09T06:19:09,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149662 2024-12-09T06:19:09,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149662 2024-12-09T06:19:09,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149662) roll requested 2024-12-09T06:19:09,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149668 2024-12-09T06:19:09,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149668 2024-12-09T06:19:09,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149668) roll requested 2024-12-09T06:19:09,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149675 2024-12-09T06:19:09,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149675 2024-12-09T06:19:09,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149675) roll requested 2024-12-09T06:19:09,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149682 2024-12-09T06:19:09,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149682 2024-12-09T06:19:09,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149682) roll requested 2024-12-09T06:19:09,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149689 2024-12-09T06:19:09,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149689 2024-12-09T06:19:09,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149689) roll requested 2024-12-09T06:19:09,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149697 2024-12-09T06:19:09,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149697 2024-12-09T06:19:09,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149697) roll requested 2024-12-09T06:19:09,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149705 2024-12-09T06:19:09,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149705 2024-12-09T06:19:09,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149705) roll requested 2024-12-09T06:19:09,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149712 2024-12-09T06:19:09,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149712 2024-12-09T06:19:09,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149712) roll requested 2024-12-09T06:19:09,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149721 2024-12-09T06:19:09,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149721 2024-12-09T06:19:09,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149721) roll requested 2024-12-09T06:19:09,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149730 2024-12-09T06:19:09,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149730 2024-12-09T06:19:09,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149730) roll requested 2024-12-09T06:19:09,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149737 2024-12-09T06:19:09,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149737 2024-12-09T06:19:09,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149737) roll requested 2024-12-09T06:19:09,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149745 2024-12-09T06:19:09,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149745 2024-12-09T06:19:09,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149745) roll requested 2024-12-09T06:19:09,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149753 2024-12-09T06:19:09,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149753 2024-12-09T06:19:09,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149753) roll requested 2024-12-09T06:19:09,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149761 2024-12-09T06:19:09,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149761 2024-12-09T06:19:09,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149761) roll requested 2024-12-09T06:19:09,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149769 2024-12-09T06:19:09,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149769 2024-12-09T06:19:09,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149769) roll requested 2024-12-09T06:19:09,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149777 2024-12-09T06:19:09,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149777 2024-12-09T06:19:09,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149777) roll requested 2024-12-09T06:19:09,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149785 2024-12-09T06:19:09,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149785 2024-12-09T06:19:09,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149785) roll requested 2024-12-09T06:19:09,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149795 2024-12-09T06:19:09,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149795 2024-12-09T06:19:09,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149795) roll requested 2024-12-09T06:19:09,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149802 2024-12-09T06:19:09,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149802 2024-12-09T06:19:09,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149802) roll requested 2024-12-09T06:19:09,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149807 2024-12-09T06:19:09,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149807 2024-12-09T06:19:09,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149807) roll requested 2024-12-09T06:19:09,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149814 2024-12-09T06:19:09,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149814 2024-12-09T06:19:09,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149814) roll requested 2024-12-09T06:19:09,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149821 2024-12-09T06:19:09,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149821 2024-12-09T06:19:09,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149821) roll requested 2024-12-09T06:19:09,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149828 2024-12-09T06:19:09,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149828 2024-12-09T06:19:09,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149828) roll requested 2024-12-09T06:19:09,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149834 2024-12-09T06:19:09,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149834 2024-12-09T06:19:09,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149834) roll requested 2024-12-09T06:19:09,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149839 2024-12-09T06:19:09,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149839 2024-12-09T06:19:09,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149839) roll requested 2024-12-09T06:19:09,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149846 2024-12-09T06:19:09,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149846 2024-12-09T06:19:09,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149846) roll requested 2024-12-09T06:19:09,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149853 2024-12-09T06:19:09,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149853 2024-12-09T06:19:09,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149853) roll requested 2024-12-09T06:19:09,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149861 2024-12-09T06:19:09,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149861 2024-12-09T06:19:09,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149861) roll requested 2024-12-09T06:19:09,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149868 2024-12-09T06:19:09,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149868 2024-12-09T06:19:09,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149868) roll requested 2024-12-09T06:19:09,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149875 2024-12-09T06:19:09,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149875 2024-12-09T06:19:09,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149875) roll requested 2024-12-09T06:19:09,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149881 2024-12-09T06:19:09,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149881 2024-12-09T06:19:09,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149881) roll requested 2024-12-09T06:19:09,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149888 2024-12-09T06:19:09,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149888 2024-12-09T06:19:09,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149888) roll requested 2024-12-09T06:19:09,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149895 2024-12-09T06:19:09,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149895 2024-12-09T06:19:09,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149895) roll requested 2024-12-09T06:19:09,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149903 2024-12-09T06:19:09,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149903 2024-12-09T06:19:09,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149903) roll requested 2024-12-09T06:19:09,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149911 2024-12-09T06:19:09,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149911 2024-12-09T06:19:09,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149911) roll requested 2024-12-09T06:19:09,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149919 2024-12-09T06:19:09,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149919 2024-12-09T06:19:09,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149919) roll requested 2024-12-09T06:19:09,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149928 2024-12-09T06:19:09,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149928 2024-12-09T06:19:09,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149928) roll requested 2024-12-09T06:19:09,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149937 2024-12-09T06:19:09,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149937 2024-12-09T06:19:09,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149937) roll requested 2024-12-09T06:19:09,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149947 2024-12-09T06:19:09,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149947 2024-12-09T06:19:09,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149947) roll requested 2024-12-09T06:19:09,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149958 2024-12-09T06:19:09,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149958 2024-12-09T06:19:09,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149958) roll requested 2024-12-09T06:19:09,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149966 2024-12-09T06:19:09,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149966 2024-12-09T06:19:09,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149966) roll requested 2024-12-09T06:19:09,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149976 2024-12-09T06:19:09,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149976 2024-12-09T06:19:09,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149976) roll requested 2024-12-09T06:19:09,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149981 2024-12-09T06:19:09,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149981 2024-12-09T06:19:09,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149981) roll requested 2024-12-09T06:19:09,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149987 2024-12-09T06:19:09,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:09,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149987 2024-12-09T06:19:09,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:09,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:09,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149987) roll requested 2024-12-09T06:19:09,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725149995 2024-12-09T06:19:10,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149995 2024-12-09T06:19:10,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725149995) roll requested 2024-12-09T06:19:10,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150002 2024-12-09T06:19:10,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725149995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150002 2024-12-09T06:19:10,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150002) roll requested 2024-12-09T06:19:10,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150009 2024-12-09T06:19:10,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150009 2024-12-09T06:19:10,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150009) roll requested 2024-12-09T06:19:10,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150014 2024-12-09T06:19:10,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150014 2024-12-09T06:19:10,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150014) roll requested 2024-12-09T06:19:10,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150022 2024-12-09T06:19:10,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150022 2024-12-09T06:19:10,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150022) roll requested 2024-12-09T06:19:10,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150027 2024-12-09T06:19:10,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150027 2024-12-09T06:19:10,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150027) roll requested 2024-12-09T06:19:10,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150032 2024-12-09T06:19:10,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150032 2024-12-09T06:19:10,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150032) roll requested 2024-12-09T06:19:10,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150038 2024-12-09T06:19:10,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150038 2024-12-09T06:19:10,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150038) roll requested 2024-12-09T06:19:10,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150044 2024-12-09T06:19:10,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150044 2024-12-09T06:19:10,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150044) roll requested 2024-12-09T06:19:10,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150049 2024-12-09T06:19:10,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150049 2024-12-09T06:19:10,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150049) roll requested 2024-12-09T06:19:10,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150056 2024-12-09T06:19:10,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150056 2024-12-09T06:19:10,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150056) roll requested 2024-12-09T06:19:10,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150062 2024-12-09T06:19:10,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150062 2024-12-09T06:19:10,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150062) roll requested 2024-12-09T06:19:10,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150068 2024-12-09T06:19:10,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150068 2024-12-09T06:19:10,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150068) roll requested 2024-12-09T06:19:10,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150073 2024-12-09T06:19:10,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150073 2024-12-09T06:19:10,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150073) roll requested 2024-12-09T06:19:10,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150079 2024-12-09T06:19:10,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150079 2024-12-09T06:19:10,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150079) roll requested 2024-12-09T06:19:10,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150084 2024-12-09T06:19:10,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150084 2024-12-09T06:19:10,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150084) roll requested 2024-12-09T06:19:10,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150089 2024-12-09T06:19:10,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150089 2024-12-09T06:19:10,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150089) roll requested 2024-12-09T06:19:10,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150094 2024-12-09T06:19:10,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150094 2024-12-09T06:19:10,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150094) roll requested 2024-12-09T06:19:10,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150100 2024-12-09T06:19:10,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150100 2024-12-09T06:19:10,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150100) roll requested 2024-12-09T06:19:10,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150105 2024-12-09T06:19:10,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150105 2024-12-09T06:19:10,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150105) roll requested 2024-12-09T06:19:10,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150110 2024-12-09T06:19:10,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150110 2024-12-09T06:19:10,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150110) roll requested 2024-12-09T06:19:10,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150116 2024-12-09T06:19:10,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150116 2024-12-09T06:19:10,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150116) roll requested 2024-12-09T06:19:10,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150121 2024-12-09T06:19:10,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150121 2024-12-09T06:19:10,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150121) roll requested 2024-12-09T06:19:10,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150127 2024-12-09T06:19:10,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150127 2024-12-09T06:19:10,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150127) roll requested 2024-12-09T06:19:10,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150133 2024-12-09T06:19:10,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150133 2024-12-09T06:19:10,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150133) roll requested 2024-12-09T06:19:10,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150142 2024-12-09T06:19:10,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150142 2024-12-09T06:19:10,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150142) roll requested 2024-12-09T06:19:10,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150149 2024-12-09T06:19:10,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150149 2024-12-09T06:19:10,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150149) roll requested 2024-12-09T06:19:10,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150154 2024-12-09T06:19:10,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150154 2024-12-09T06:19:10,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150154) roll requested 2024-12-09T06:19:10,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150159 2024-12-09T06:19:10,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150159 2024-12-09T06:19:10,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150159) roll requested 2024-12-09T06:19:10,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150166 2024-12-09T06:19:10,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150166 2024-12-09T06:19:10,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150166) roll requested 2024-12-09T06:19:10,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150173 2024-12-09T06:19:10,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150173 2024-12-09T06:19:10,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150173) roll requested 2024-12-09T06:19:10,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150180 2024-12-09T06:19:10,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150180 2024-12-09T06:19:10,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150180) roll requested 2024-12-09T06:19:10,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150188 2024-12-09T06:19:10,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150188 2024-12-09T06:19:10,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150188) roll requested 2024-12-09T06:19:10,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150194 2024-12-09T06:19:10,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150194 2024-12-09T06:19:10,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150194) roll requested 2024-12-09T06:19:10,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150200 2024-12-09T06:19:10,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150200 2024-12-09T06:19:10,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150200) roll requested 2024-12-09T06:19:10,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150207 2024-12-09T06:19:10,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150207 2024-12-09T06:19:10,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150207) roll requested 2024-12-09T06:19:10,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150214 2024-12-09T06:19:10,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150214 2024-12-09T06:19:10,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150214) roll requested 2024-12-09T06:19:10,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150223 2024-12-09T06:19:10,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150223 2024-12-09T06:19:10,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150223) roll requested 2024-12-09T06:19:10,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150230 2024-12-09T06:19:10,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150230 2024-12-09T06:19:10,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150230) roll requested 2024-12-09T06:19:10,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150236 2024-12-09T06:19:10,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150236 2024-12-09T06:19:10,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150236) roll requested 2024-12-09T06:19:10,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150244 2024-12-09T06:19:10,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150244 2024-12-09T06:19:10,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150244) roll requested 2024-12-09T06:19:10,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150252 2024-12-09T06:19:10,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150252 2024-12-09T06:19:10,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150252) roll requested 2024-12-09T06:19:10,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150259 2024-12-09T06:19:10,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150259 2024-12-09T06:19:10,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150259) roll requested 2024-12-09T06:19:10,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150268 2024-12-09T06:19:10,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150268 2024-12-09T06:19:10,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150268) roll requested 2024-12-09T06:19:10,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150274 2024-12-09T06:19:10,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150274 2024-12-09T06:19:10,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150274) roll requested 2024-12-09T06:19:10,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150283 2024-12-09T06:19:10,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150283 2024-12-09T06:19:10,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150283) roll requested 2024-12-09T06:19:10,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150289 2024-12-09T06:19:10,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150289 2024-12-09T06:19:10,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150289) roll requested 2024-12-09T06:19:10,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150294 2024-12-09T06:19:10,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150294 2024-12-09T06:19:10,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150294) roll requested 2024-12-09T06:19:10,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150299 2024-12-09T06:19:10,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150299 2024-12-09T06:19:10,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150299) roll requested 2024-12-09T06:19:10,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150304 2024-12-09T06:19:10,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150304 2024-12-09T06:19:10,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150304) roll requested 2024-12-09T06:19:10,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150311 2024-12-09T06:19:10,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150311 2024-12-09T06:19:10,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150311) roll requested 2024-12-09T06:19:10,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150322 2024-12-09T06:19:10,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150322 2024-12-09T06:19:10,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150322) roll requested 2024-12-09T06:19:10,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150329 2024-12-09T06:19:10,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150329 2024-12-09T06:19:10,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150329) roll requested 2024-12-09T06:19:10,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150337 2024-12-09T06:19:10,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150337 2024-12-09T06:19:10,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150337) roll requested 2024-12-09T06:19:10,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150343 2024-12-09T06:19:10,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150343 2024-12-09T06:19:10,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150343) roll requested 2024-12-09T06:19:10,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150350 2024-12-09T06:19:10,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150350 2024-12-09T06:19:10,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150350) roll requested 2024-12-09T06:19:10,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150362 2024-12-09T06:19:10,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150362 2024-12-09T06:19:10,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150362) roll requested 2024-12-09T06:19:10,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150370 2024-12-09T06:19:10,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150370 2024-12-09T06:19:10,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150370) roll requested 2024-12-09T06:19:10,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150377 2024-12-09T06:19:10,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150377 2024-12-09T06:19:10,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150377) roll requested 2024-12-09T06:19:10,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150384 2024-12-09T06:19:10,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150384 2024-12-09T06:19:10,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150384) roll requested 2024-12-09T06:19:10,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150390 2024-12-09T06:19:10,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150390 2024-12-09T06:19:10,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150390) roll requested 2024-12-09T06:19:10,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150401 2024-12-09T06:19:10,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150401 2024-12-09T06:19:10,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150401) roll requested 2024-12-09T06:19:10,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150414 2024-12-09T06:19:10,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150414 2024-12-09T06:19:10,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150414) roll requested 2024-12-09T06:19:10,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150425 2024-12-09T06:19:10,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150425 2024-12-09T06:19:10,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150425) roll requested 2024-12-09T06:19:10,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150439 2024-12-09T06:19:10,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150439 2024-12-09T06:19:10,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150439) roll requested 2024-12-09T06:19:10,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150445 2024-12-09T06:19:10,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150445 2024-12-09T06:19:10,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150445) roll requested 2024-12-09T06:19:10,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150452 2024-12-09T06:19:10,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150452 2024-12-09T06:19:10,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150452) roll requested 2024-12-09T06:19:10,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150457 2024-12-09T06:19:10,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150457 2024-12-09T06:19:10,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150457) roll requested 2024-12-09T06:19:10,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150462 2024-12-09T06:19:10,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150462 2024-12-09T06:19:10,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150462) roll requested 2024-12-09T06:19:10,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150473 2024-12-09T06:19:10,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150473 2024-12-09T06:19:10,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150473) roll requested 2024-12-09T06:19:10,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150479 2024-12-09T06:19:10,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150479 2024-12-09T06:19:10,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150479) roll requested 2024-12-09T06:19:10,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150486 2024-12-09T06:19:10,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150486 2024-12-09T06:19:10,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150486) roll requested 2024-12-09T06:19:10,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150491 2024-12-09T06:19:10,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150491 2024-12-09T06:19:10,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150491) roll requested 2024-12-09T06:19:10,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150497 2024-12-09T06:19:10,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150497 2024-12-09T06:19:10,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150497) roll requested 2024-12-09T06:19:10,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150502 2024-12-09T06:19:10,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150502 2024-12-09T06:19:10,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150502) roll requested 2024-12-09T06:19:10,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150507 2024-12-09T06:19:10,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150507 2024-12-09T06:19:10,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150507) roll requested 2024-12-09T06:19:10,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150513 2024-12-09T06:19:10,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150513 2024-12-09T06:19:10,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150513) roll requested 2024-12-09T06:19:10,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150520 2024-12-09T06:19:10,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150520 2024-12-09T06:19:10,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150520) roll requested 2024-12-09T06:19:10,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150527 2024-12-09T06:19:10,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150527 2024-12-09T06:19:10,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150527) roll requested 2024-12-09T06:19:10,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150537 2024-12-09T06:19:10,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150537 2024-12-09T06:19:10,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150537) roll requested 2024-12-09T06:19:10,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150543 2024-12-09T06:19:10,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150543 2024-12-09T06:19:10,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150543) roll requested 2024-12-09T06:19:10,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150549 2024-12-09T06:19:10,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150549 2024-12-09T06:19:10,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150549) roll requested 2024-12-09T06:19:10,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150557 2024-12-09T06:19:10,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150557 2024-12-09T06:19:10,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150557) roll requested 2024-12-09T06:19:10,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150564 2024-12-09T06:19:10,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150564 2024-12-09T06:19:10,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150564) roll requested 2024-12-09T06:19:10,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150570 2024-12-09T06:19:10,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150570 2024-12-09T06:19:10,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150570) roll requested 2024-12-09T06:19:10,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150579 2024-12-09T06:19:10,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150579 2024-12-09T06:19:10,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150579) roll requested 2024-12-09T06:19:10,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150587 2024-12-09T06:19:10,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150587 2024-12-09T06:19:10,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150587) roll requested 2024-12-09T06:19:10,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150596 2024-12-09T06:19:10,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150596 2024-12-09T06:19:10,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150596) roll requested 2024-12-09T06:19:10,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150606 2024-12-09T06:19:10,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150606 2024-12-09T06:19:10,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150606) roll requested 2024-12-09T06:19:10,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150614 2024-12-09T06:19:10,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150614 2024-12-09T06:19:10,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150614) roll requested 2024-12-09T06:19:10,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150621 2024-12-09T06:19:10,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150621 2024-12-09T06:19:10,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150621) roll requested 2024-12-09T06:19:10,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150631 2024-12-09T06:19:10,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150631 2024-12-09T06:19:10,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150631) roll requested 2024-12-09T06:19:10,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150640 2024-12-09T06:19:10,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150640 2024-12-09T06:19:10,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150640) roll requested 2024-12-09T06:19:10,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150647 2024-12-09T06:19:10,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150647 2024-12-09T06:19:10,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150647) roll requested 2024-12-09T06:19:10,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150655 2024-12-09T06:19:10,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150655 2024-12-09T06:19:10,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150655) roll requested 2024-12-09T06:19:10,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150664 2024-12-09T06:19:10,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150664 2024-12-09T06:19:10,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150664) roll requested 2024-12-09T06:19:10,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150672 2024-12-09T06:19:10,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150672 2024-12-09T06:19:10,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150672) roll requested 2024-12-09T06:19:10,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150679 2024-12-09T06:19:10,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150679 2024-12-09T06:19:10,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150679) roll requested 2024-12-09T06:19:10,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150686 2024-12-09T06:19:10,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150686 2024-12-09T06:19:10,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150686) roll requested 2024-12-09T06:19:10,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150693 2024-12-09T06:19:10,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150693 2024-12-09T06:19:10,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150693) roll requested 2024-12-09T06:19:10,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150700 2024-12-09T06:19:10,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150700 2024-12-09T06:19:10,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150700) roll requested 2024-12-09T06:19:10,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150708 2024-12-09T06:19:10,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150708 2024-12-09T06:19:10,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150708) roll requested 2024-12-09T06:19:10,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150716 2024-12-09T06:19:10,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150716 2024-12-09T06:19:10,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150716) roll requested 2024-12-09T06:19:10,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150723 2024-12-09T06:19:10,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150723 2024-12-09T06:19:10,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150723) roll requested 2024-12-09T06:19:10,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150730 2024-12-09T06:19:10,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150730 2024-12-09T06:19:10,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150730) roll requested 2024-12-09T06:19:10,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150737 2024-12-09T06:19:10,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150737 2024-12-09T06:19:10,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150737) roll requested 2024-12-09T06:19:10,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150746 2024-12-09T06:19:10,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150746 2024-12-09T06:19:10,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150746) roll requested 2024-12-09T06:19:10,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150753 2024-12-09T06:19:10,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150753 2024-12-09T06:19:10,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150753) roll requested 2024-12-09T06:19:10,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150760 2024-12-09T06:19:10,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150760 2024-12-09T06:19:10,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150760) roll requested 2024-12-09T06:19:10,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150769 2024-12-09T06:19:10,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150769 2024-12-09T06:19:10,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150769) roll requested 2024-12-09T06:19:10,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150777 2024-12-09T06:19:10,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150777 2024-12-09T06:19:10,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150777) roll requested 2024-12-09T06:19:10,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150784 2024-12-09T06:19:10,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150784 2024-12-09T06:19:10,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150784) roll requested 2024-12-09T06:19:10,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150791 2024-12-09T06:19:10,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150791 2024-12-09T06:19:10,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150791) roll requested 2024-12-09T06:19:10,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150799 2024-12-09T06:19:10,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150799 2024-12-09T06:19:10,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150799) roll requested 2024-12-09T06:19:10,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150809 2024-12-09T06:19:10,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150809 2024-12-09T06:19:10,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150809) roll requested 2024-12-09T06:19:10,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150819 2024-12-09T06:19:10,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150819 2024-12-09T06:19:10,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150819) roll requested 2024-12-09T06:19:10,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150825 2024-12-09T06:19:10,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150825 2024-12-09T06:19:10,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150825) roll requested 2024-12-09T06:19:10,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150830 2024-12-09T06:19:10,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150830 2024-12-09T06:19:10,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150830) roll requested 2024-12-09T06:19:10,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150834 2024-12-09T06:19:10,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150834 2024-12-09T06:19:10,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150834) roll requested 2024-12-09T06:19:10,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150840 2024-12-09T06:19:10,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150840 2024-12-09T06:19:10,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150840) roll requested 2024-12-09T06:19:10,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150848 2024-12-09T06:19:10,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150848 2024-12-09T06:19:10,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150848) roll requested 2024-12-09T06:19:10,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150857 2024-12-09T06:19:10,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150857 2024-12-09T06:19:10,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150857) roll requested 2024-12-09T06:19:10,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150864 2024-12-09T06:19:10,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150864 2024-12-09T06:19:10,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150864) roll requested 2024-12-09T06:19:10,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150872 2024-12-09T06:19:10,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150872 2024-12-09T06:19:10,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150872) roll requested 2024-12-09T06:19:10,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150877 2024-12-09T06:19:10,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150877 2024-12-09T06:19:10,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150877) roll requested 2024-12-09T06:19:10,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150882 2024-12-09T06:19:10,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150882 2024-12-09T06:19:10,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150882) roll requested 2024-12-09T06:19:10,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150888 2024-12-09T06:19:10,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150888 2024-12-09T06:19:10,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150888) roll requested 2024-12-09T06:19:10,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150894 2024-12-09T06:19:10,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150894 2024-12-09T06:19:10,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150894) roll requested 2024-12-09T06:19:10,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150902 2024-12-09T06:19:10,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150902 2024-12-09T06:19:10,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150902) roll requested 2024-12-09T06:19:10,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150908 2024-12-09T06:19:10,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150908 2024-12-09T06:19:10,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150908) roll requested 2024-12-09T06:19:10,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150913 2024-12-09T06:19:10,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150913 2024-12-09T06:19:10,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150913) roll requested 2024-12-09T06:19:10,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150919 2024-12-09T06:19:10,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150919 2024-12-09T06:19:10,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150919) roll requested 2024-12-09T06:19:10,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150926 2024-12-09T06:19:10,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150926 2024-12-09T06:19:10,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150926) roll requested 2024-12-09T06:19:10,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150932 2024-12-09T06:19:10,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150932 2024-12-09T06:19:10,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150932) roll requested 2024-12-09T06:19:10,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150941 2024-12-09T06:19:10,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150941 2024-12-09T06:19:10,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150941) roll requested 2024-12-09T06:19:10,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150947 2024-12-09T06:19:10,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150947 2024-12-09T06:19:10,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150947) roll requested 2024-12-09T06:19:10,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150953 2024-12-09T06:19:10,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150953 2024-12-09T06:19:10,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150953) roll requested 2024-12-09T06:19:10,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150958 2024-12-09T06:19:10,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150958 2024-12-09T06:19:10,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150958) roll requested 2024-12-09T06:19:10,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150965 2024-12-09T06:19:10,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150965 2024-12-09T06:19:10,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150965) roll requested 2024-12-09T06:19:10,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150971 2024-12-09T06:19:10,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150971 2024-12-09T06:19:10,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150971) roll requested 2024-12-09T06:19:10,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150977 2024-12-09T06:19:10,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150977 2024-12-09T06:19:10,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150977) roll requested 2024-12-09T06:19:10,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150985 2024-12-09T06:19:10,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150985 2024-12-09T06:19:10,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150985) roll requested 2024-12-09T06:19:10,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150990 2024-12-09T06:19:10,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:10,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150990 2024-12-09T06:19:10,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:10,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:10,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150990) roll requested 2024-12-09T06:19:10,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725150996 2024-12-09T06:19:11,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150996 2024-12-09T06:19:11,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725150996) roll requested 2024-12-09T06:19:11,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151002 2024-12-09T06:19:11,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725150996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151002 2024-12-09T06:19:11,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151002) roll requested 2024-12-09T06:19:11,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151007 2024-12-09T06:19:11,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151007 2024-12-09T06:19:11,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151007) roll requested 2024-12-09T06:19:11,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151011 2024-12-09T06:19:11,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151011 2024-12-09T06:19:11,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151011) roll requested 2024-12-09T06:19:11,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151017 2024-12-09T06:19:11,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151017 2024-12-09T06:19:11,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151017) roll requested 2024-12-09T06:19:11,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151024 2024-12-09T06:19:11,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151024 2024-12-09T06:19:11,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151024) roll requested 2024-12-09T06:19:11,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151032 2024-12-09T06:19:11,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151032 2024-12-09T06:19:11,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151032) roll requested 2024-12-09T06:19:11,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151039 2024-12-09T06:19:11,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151039 2024-12-09T06:19:11,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151039) roll requested 2024-12-09T06:19:11,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151047 2024-12-09T06:19:11,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151047 2024-12-09T06:19:11,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151047) roll requested 2024-12-09T06:19:11,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151058 2024-12-09T06:19:11,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151058 2024-12-09T06:19:11,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151058) roll requested 2024-12-09T06:19:11,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151063 2024-12-09T06:19:11,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151063 2024-12-09T06:19:11,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151063) roll requested 2024-12-09T06:19:11,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151068 2024-12-09T06:19:11,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151068 2024-12-09T06:19:11,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151068) roll requested 2024-12-09T06:19:11,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151073 2024-12-09T06:19:11,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151073 2024-12-09T06:19:11,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151073) roll requested 2024-12-09T06:19:11,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151078 2024-12-09T06:19:11,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151078 2024-12-09T06:19:11,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151078) roll requested 2024-12-09T06:19:11,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151083 2024-12-09T06:19:11,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151083 2024-12-09T06:19:11,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151083) roll requested 2024-12-09T06:19:11,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151089 2024-12-09T06:19:11,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151089 2024-12-09T06:19:11,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151089) roll requested 2024-12-09T06:19:11,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151097 2024-12-09T06:19:11,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151097 2024-12-09T06:19:11,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151097) roll requested 2024-12-09T06:19:11,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151104 2024-12-09T06:19:11,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151104 2024-12-09T06:19:11,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151104) roll requested 2024-12-09T06:19:11,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151111 2024-12-09T06:19:11,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151111 2024-12-09T06:19:11,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151111) roll requested 2024-12-09T06:19:11,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151119 2024-12-09T06:19:11,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151119 2024-12-09T06:19:11,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151119) roll requested 2024-12-09T06:19:11,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151125 2024-12-09T06:19:11,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151125 2024-12-09T06:19:11,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151125) roll requested 2024-12-09T06:19:11,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151133 2024-12-09T06:19:11,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151133 2024-12-09T06:19:11,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151133) roll requested 2024-12-09T06:19:11,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151142 2024-12-09T06:19:11,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151142 2024-12-09T06:19:11,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151142) roll requested 2024-12-09T06:19:11,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151151 2024-12-09T06:19:11,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151151 2024-12-09T06:19:11,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151151) roll requested 2024-12-09T06:19:11,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151158 2024-12-09T06:19:11,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151158 2024-12-09T06:19:11,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151158) roll requested 2024-12-09T06:19:11,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151165 2024-12-09T06:19:11,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151165 2024-12-09T06:19:11,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151165) roll requested 2024-12-09T06:19:11,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151173 2024-12-09T06:19:11,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151173 2024-12-09T06:19:11,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151173) roll requested 2024-12-09T06:19:11,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151178 2024-12-09T06:19:11,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151178 2024-12-09T06:19:11,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151178) roll requested 2024-12-09T06:19:11,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151183 2024-12-09T06:19:11,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151183 2024-12-09T06:19:11,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151183) roll requested 2024-12-09T06:19:11,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151189 2024-12-09T06:19:11,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151189 2024-12-09T06:19:11,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151189) roll requested 2024-12-09T06:19:11,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151195 2024-12-09T06:19:11,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151195 2024-12-09T06:19:11,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151195) roll requested 2024-12-09T06:19:11,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151203 2024-12-09T06:19:11,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151203 2024-12-09T06:19:11,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151203) roll requested 2024-12-09T06:19:11,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151210 2024-12-09T06:19:11,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151210 2024-12-09T06:19:11,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151210) roll requested 2024-12-09T06:19:11,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151219 2024-12-09T06:19:11,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151219 2024-12-09T06:19:11,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151219) roll requested 2024-12-09T06:19:11,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151233 2024-12-09T06:19:11,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151233 2024-12-09T06:19:11,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151233) roll requested 2024-12-09T06:19:11,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151238 2024-12-09T06:19:11,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151238 2024-12-09T06:19:11,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151238) roll requested 2024-12-09T06:19:11,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151243 2024-12-09T06:19:11,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151243 2024-12-09T06:19:11,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151243) roll requested 2024-12-09T06:19:11,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151250 2024-12-09T06:19:11,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151250 2024-12-09T06:19:11,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151250) roll requested 2024-12-09T06:19:11,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151257 2024-12-09T06:19:11,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151257 2024-12-09T06:19:11,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151257) roll requested 2024-12-09T06:19:11,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151266 2024-12-09T06:19:11,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151266 2024-12-09T06:19:11,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151266) roll requested 2024-12-09T06:19:11,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151273 2024-12-09T06:19:11,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151273 2024-12-09T06:19:11,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151273) roll requested 2024-12-09T06:19:11,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151285 2024-12-09T06:19:11,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151285 2024-12-09T06:19:11,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151285) roll requested 2024-12-09T06:19:11,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151293 2024-12-09T06:19:11,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151293 2024-12-09T06:19:11,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151293) roll requested 2024-12-09T06:19:11,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151298 2024-12-09T06:19:11,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151298 2024-12-09T06:19:11,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151298) roll requested 2024-12-09T06:19:11,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151303 2024-12-09T06:19:11,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151303 2024-12-09T06:19:11,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151303) roll requested 2024-12-09T06:19:11,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151309 2024-12-09T06:19:11,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151309 2024-12-09T06:19:11,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151309) roll requested 2024-12-09T06:19:11,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151314 2024-12-09T06:19:11,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151314 2024-12-09T06:19:11,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151314) roll requested 2024-12-09T06:19:11,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151319 2024-12-09T06:19:11,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151319 2024-12-09T06:19:11,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151319) roll requested 2024-12-09T06:19:11,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151324 2024-12-09T06:19:11,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151324 2024-12-09T06:19:11,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151324) roll requested 2024-12-09T06:19:11,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151330 2024-12-09T06:19:11,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151330 2024-12-09T06:19:11,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151330) roll requested 2024-12-09T06:19:11,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151337 2024-12-09T06:19:11,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151337 2024-12-09T06:19:11,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151337) roll requested 2024-12-09T06:19:11,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151344 2024-12-09T06:19:11,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151344 2024-12-09T06:19:11,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151344) roll requested 2024-12-09T06:19:11,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151352 2024-12-09T06:19:11,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151352 2024-12-09T06:19:11,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151352) roll requested 2024-12-09T06:19:11,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151361 2024-12-09T06:19:11,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151361 2024-12-09T06:19:11,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151361) roll requested 2024-12-09T06:19:11,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151371 2024-12-09T06:19:11,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151371 2024-12-09T06:19:11,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151371) roll requested 2024-12-09T06:19:11,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151377 2024-12-09T06:19:11,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151377 2024-12-09T06:19:11,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151377) roll requested 2024-12-09T06:19:11,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151383 2024-12-09T06:19:11,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151383 2024-12-09T06:19:11,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151383) roll requested 2024-12-09T06:19:11,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151389 2024-12-09T06:19:11,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151389 2024-12-09T06:19:11,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151389) roll requested 2024-12-09T06:19:11,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151397 2024-12-09T06:19:11,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151397 2024-12-09T06:19:11,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151397) roll requested 2024-12-09T06:19:11,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151404 2024-12-09T06:19:11,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151404 2024-12-09T06:19:11,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151404) roll requested 2024-12-09T06:19:11,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151412 2024-12-09T06:19:11,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151412 2024-12-09T06:19:11,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151412) roll requested 2024-12-09T06:19:11,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151421 2024-12-09T06:19:11,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151421 2024-12-09T06:19:11,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151421) roll requested 2024-12-09T06:19:11,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151428 2024-12-09T06:19:11,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151428 2024-12-09T06:19:11,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151428) roll requested 2024-12-09T06:19:11,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151434 2024-12-09T06:19:11,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151434 2024-12-09T06:19:11,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151434) roll requested 2024-12-09T06:19:11,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151444 2024-12-09T06:19:11,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151444 2024-12-09T06:19:11,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151444) roll requested 2024-12-09T06:19:11,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151452 2024-12-09T06:19:11,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151452 2024-12-09T06:19:11,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151452) roll requested 2024-12-09T06:19:11,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151460 2024-12-09T06:19:11,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151460 2024-12-09T06:19:11,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151460) roll requested 2024-12-09T06:19:11,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151465 2024-12-09T06:19:11,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151465 2024-12-09T06:19:11,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151465) roll requested 2024-12-09T06:19:11,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151471 2024-12-09T06:19:11,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151471 2024-12-09T06:19:11,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151471) roll requested 2024-12-09T06:19:11,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151478 2024-12-09T06:19:11,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151478 2024-12-09T06:19:11,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151478) roll requested 2024-12-09T06:19:11,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151483 2024-12-09T06:19:11,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151483 2024-12-09T06:19:11,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151483) roll requested 2024-12-09T06:19:11,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151488 2024-12-09T06:19:11,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151488 2024-12-09T06:19:11,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151488) roll requested 2024-12-09T06:19:11,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151494 2024-12-09T06:19:11,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151494 2024-12-09T06:19:11,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151494) roll requested 2024-12-09T06:19:11,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151500 2024-12-09T06:19:11,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151500 2024-12-09T06:19:11,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151500) roll requested 2024-12-09T06:19:11,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151507 2024-12-09T06:19:11,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151507 2024-12-09T06:19:11,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151507) roll requested 2024-12-09T06:19:11,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151515 2024-12-09T06:19:11,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151515 2024-12-09T06:19:11,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151515) roll requested 2024-12-09T06:19:11,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151523 2024-12-09T06:19:11,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151523 2024-12-09T06:19:11,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151523) roll requested 2024-12-09T06:19:11,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151533 2024-12-09T06:19:11,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151533 2024-12-09T06:19:11,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151533) roll requested 2024-12-09T06:19:11,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151542 2024-12-09T06:19:11,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151542 2024-12-09T06:19:11,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151542) roll requested 2024-12-09T06:19:11,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151548 2024-12-09T06:19:11,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151548 2024-12-09T06:19:11,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151548) roll requested 2024-12-09T06:19:11,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151554 2024-12-09T06:19:11,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151554 2024-12-09T06:19:11,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151554) roll requested 2024-12-09T06:19:11,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151562 2024-12-09T06:19:11,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151562 2024-12-09T06:19:11,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151562) roll requested 2024-12-09T06:19:11,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151570 2024-12-09T06:19:11,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151570 2024-12-09T06:19:11,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151570) roll requested 2024-12-09T06:19:11,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151579 2024-12-09T06:19:11,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151579 2024-12-09T06:19:11,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151579) roll requested 2024-12-09T06:19:11,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151585 2024-12-09T06:19:11,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151585 2024-12-09T06:19:11,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151585) roll requested 2024-12-09T06:19:11,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151592 2024-12-09T06:19:11,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151592 2024-12-09T06:19:11,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151592) roll requested 2024-12-09T06:19:11,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151600 2024-12-09T06:19:11,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151600 2024-12-09T06:19:11,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151600) roll requested 2024-12-09T06:19:11,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151608 2024-12-09T06:19:11,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151608 2024-12-09T06:19:11,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151608) roll requested 2024-12-09T06:19:11,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151615 2024-12-09T06:19:11,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151615 2024-12-09T06:19:11,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151615) roll requested 2024-12-09T06:19:11,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151623 2024-12-09T06:19:11,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151623 2024-12-09T06:19:11,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151623) roll requested 2024-12-09T06:19:11,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151631 2024-12-09T06:19:11,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151631 2024-12-09T06:19:11,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151631) roll requested 2024-12-09T06:19:11,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151641 2024-12-09T06:19:11,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151641 2024-12-09T06:19:11,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151641) roll requested 2024-12-09T06:19:11,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151648 2024-12-09T06:19:11,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151648 2024-12-09T06:19:11,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151648) roll requested 2024-12-09T06:19:11,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151655 2024-12-09T06:19:11,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151655 2024-12-09T06:19:11,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151655) roll requested 2024-12-09T06:19:11,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151662 2024-12-09T06:19:11,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151662 2024-12-09T06:19:11,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151662) roll requested 2024-12-09T06:19:11,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151669 2024-12-09T06:19:11,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151669 2024-12-09T06:19:11,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151669) roll requested 2024-12-09T06:19:11,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151679 2024-12-09T06:19:11,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151679 2024-12-09T06:19:11,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151679) roll requested 2024-12-09T06:19:11,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151689 2024-12-09T06:19:11,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151689 2024-12-09T06:19:11,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151689) roll requested 2024-12-09T06:19:11,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151696 2024-12-09T06:19:11,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151696 2024-12-09T06:19:11,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151696) roll requested 2024-12-09T06:19:11,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151701 2024-12-09T06:19:11,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151701 2024-12-09T06:19:11,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151701) roll requested 2024-12-09T06:19:11,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151705 2024-12-09T06:19:11,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151705 2024-12-09T06:19:11,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151705) roll requested 2024-12-09T06:19:11,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151711 2024-12-09T06:19:11,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151711 2024-12-09T06:19:11,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151711) roll requested 2024-12-09T06:19:11,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151718 2024-12-09T06:19:11,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151718 2024-12-09T06:19:11,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151718) roll requested 2024-12-09T06:19:11,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151725 2024-12-09T06:19:11,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151725 2024-12-09T06:19:11,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151725) roll requested 2024-12-09T06:19:11,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151732 2024-12-09T06:19:11,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151732 2024-12-09T06:19:11,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151732) roll requested 2024-12-09T06:19:11,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151742 2024-12-09T06:19:11,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151742 2024-12-09T06:19:11,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151742) roll requested 2024-12-09T06:19:11,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151750 2024-12-09T06:19:11,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151750 2024-12-09T06:19:11,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151750) roll requested 2024-12-09T06:19:11,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151758 2024-12-09T06:19:11,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151758 2024-12-09T06:19:11,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151758) roll requested 2024-12-09T06:19:11,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151765 2024-12-09T06:19:11,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151765 2024-12-09T06:19:11,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151765) roll requested 2024-12-09T06:19:11,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151774 2024-12-09T06:19:11,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151774 2024-12-09T06:19:11,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151774) roll requested 2024-12-09T06:19:11,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151781 2024-12-09T06:19:11,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151781 2024-12-09T06:19:11,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151781) roll requested 2024-12-09T06:19:11,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151789 2024-12-09T06:19:11,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151789 2024-12-09T06:19:11,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151789) roll requested 2024-12-09T06:19:11,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151795 2024-12-09T06:19:11,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151795 2024-12-09T06:19:11,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151795) roll requested 2024-12-09T06:19:11,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151802 2024-12-09T06:19:11,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151802 2024-12-09T06:19:11,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151802) roll requested 2024-12-09T06:19:11,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151808 2024-12-09T06:19:11,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151808 2024-12-09T06:19:11,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151808) roll requested 2024-12-09T06:19:11,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151816 2024-12-09T06:19:11,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151816 2024-12-09T06:19:11,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151816) roll requested 2024-12-09T06:19:11,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151823 2024-12-09T06:19:11,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151823 2024-12-09T06:19:11,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151823) roll requested 2024-12-09T06:19:11,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151830 2024-12-09T06:19:11,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151830 2024-12-09T06:19:11,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151830) roll requested 2024-12-09T06:19:11,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151838 2024-12-09T06:19:11,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151838 2024-12-09T06:19:11,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151838) roll requested 2024-12-09T06:19:11,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151847 2024-12-09T06:19:11,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151847 2024-12-09T06:19:11,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151847) roll requested 2024-12-09T06:19:11,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151855 2024-12-09T06:19:11,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151855 2024-12-09T06:19:11,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151855) roll requested 2024-12-09T06:19:11,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151863 2024-12-09T06:19:11,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151863 2024-12-09T06:19:11,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151863) roll requested 2024-12-09T06:19:11,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151871 2024-12-09T06:19:11,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151871 2024-12-09T06:19:11,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151871) roll requested 2024-12-09T06:19:11,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151880 2024-12-09T06:19:11,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151880 2024-12-09T06:19:11,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151880) roll requested 2024-12-09T06:19:11,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151893 2024-12-09T06:19:11,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151893 2024-12-09T06:19:11,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151893) roll requested 2024-12-09T06:19:11,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151900 2024-12-09T06:19:11,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151900 2024-12-09T06:19:11,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151900) roll requested 2024-12-09T06:19:11,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151904 2024-12-09T06:19:11,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151904 2024-12-09T06:19:11,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151904) roll requested 2024-12-09T06:19:11,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151909 2024-12-09T06:19:11,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151909 2024-12-09T06:19:11,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151909) roll requested 2024-12-09T06:19:11,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151915 2024-12-09T06:19:11,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151915 2024-12-09T06:19:11,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151915) roll requested 2024-12-09T06:19:11,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151922 2024-12-09T06:19:11,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151922 2024-12-09T06:19:11,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151922) roll requested 2024-12-09T06:19:11,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151929 2024-12-09T06:19:11,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151929 2024-12-09T06:19:11,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151929) roll requested 2024-12-09T06:19:11,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151936 2024-12-09T06:19:11,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151936 2024-12-09T06:19:11,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151936) roll requested 2024-12-09T06:19:11,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151944 2024-12-09T06:19:11,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151944 2024-12-09T06:19:11,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151944) roll requested 2024-12-09T06:19:11,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151952 2024-12-09T06:19:11,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151952 2024-12-09T06:19:11,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151952) roll requested 2024-12-09T06:19:11,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151961 2024-12-09T06:19:11,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151961 2024-12-09T06:19:11,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151961) roll requested 2024-12-09T06:19:11,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151973 2024-12-09T06:19:11,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151973 2024-12-09T06:19:11,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151973) roll requested 2024-12-09T06:19:11,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151978 2024-12-09T06:19:11,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151978 2024-12-09T06:19:11,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151978) roll requested 2024-12-09T06:19:11,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151983 2024-12-09T06:19:11,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151983 2024-12-09T06:19:11,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151983) roll requested 2024-12-09T06:19:11,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151988 2024-12-09T06:19:11,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:11,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151988 2024-12-09T06:19:11,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:11,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:11,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151988) roll requested 2024-12-09T06:19:11,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725151994 2024-12-09T06:19:12,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151994 2024-12-09T06:19:12,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725151994) roll requested 2024-12-09T06:19:12,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152005 2024-12-09T06:19:12,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725151994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152005 2024-12-09T06:19:12,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152005) roll requested 2024-12-09T06:19:12,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152011 2024-12-09T06:19:12,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152011 2024-12-09T06:19:12,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152011) roll requested 2024-12-09T06:19:12,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152016 2024-12-09T06:19:12,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152016 2024-12-09T06:19:12,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152016) roll requested 2024-12-09T06:19:12,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152022 2024-12-09T06:19:12,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152022 2024-12-09T06:19:12,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152022) roll requested 2024-12-09T06:19:12,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152027 2024-12-09T06:19:12,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152027 2024-12-09T06:19:12,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152027) roll requested 2024-12-09T06:19:12,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152034 2024-12-09T06:19:12,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152034 2024-12-09T06:19:12,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152034) roll requested 2024-12-09T06:19:12,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152041 2024-12-09T06:19:12,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152041 2024-12-09T06:19:12,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152041) roll requested 2024-12-09T06:19:12,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152049 2024-12-09T06:19:12,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152049 2024-12-09T06:19:12,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152049) roll requested 2024-12-09T06:19:12,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152057 2024-12-09T06:19:12,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152057 2024-12-09T06:19:12,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152057) roll requested 2024-12-09T06:19:12,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152065 2024-12-09T06:19:12,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152065 2024-12-09T06:19:12,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152065) roll requested 2024-12-09T06:19:12,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152074 2024-12-09T06:19:12,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152074 2024-12-09T06:19:12,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152074) roll requested 2024-12-09T06:19:12,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152082 2024-12-09T06:19:12,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152082 2024-12-09T06:19:12,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152082) roll requested 2024-12-09T06:19:12,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152091 2024-12-09T06:19:12,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152091 2024-12-09T06:19:12,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152091) roll requested 2024-12-09T06:19:12,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152100 2024-12-09T06:19:12,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152100 2024-12-09T06:19:12,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152100) roll requested 2024-12-09T06:19:12,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152107 2024-12-09T06:19:12,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152107 2024-12-09T06:19:12,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152107) roll requested 2024-12-09T06:19:12,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152114 2024-12-09T06:19:12,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152114 2024-12-09T06:19:12,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152114) roll requested 2024-12-09T06:19:12,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152122 2024-12-09T06:19:12,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152122 2024-12-09T06:19:12,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152122) roll requested 2024-12-09T06:19:12,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152130 2024-12-09T06:19:12,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152130 2024-12-09T06:19:12,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152130) roll requested 2024-12-09T06:19:12,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152139 2024-12-09T06:19:12,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152139 2024-12-09T06:19:12,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152139) roll requested 2024-12-09T06:19:12,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152146 2024-12-09T06:19:12,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152146 2024-12-09T06:19:12,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152146) roll requested 2024-12-09T06:19:12,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152152 2024-12-09T06:19:12,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152152 2024-12-09T06:19:12,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152152) roll requested 2024-12-09T06:19:12,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152158 2024-12-09T06:19:12,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152158 2024-12-09T06:19:12,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152158) roll requested 2024-12-09T06:19:12,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152164 2024-12-09T06:19:12,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152164 2024-12-09T06:19:12,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152164) roll requested 2024-12-09T06:19:12,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152172 2024-12-09T06:19:12,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152172 2024-12-09T06:19:12,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152172) roll requested 2024-12-09T06:19:12,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152181 2024-12-09T06:19:12,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152181 2024-12-09T06:19:12,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152181) roll requested 2024-12-09T06:19:12,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152190 2024-12-09T06:19:12,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152190 2024-12-09T06:19:12,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152190) roll requested 2024-12-09T06:19:12,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152198 2024-12-09T06:19:12,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152198 2024-12-09T06:19:12,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152198) roll requested 2024-12-09T06:19:12,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152207 2024-12-09T06:19:12,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152207 2024-12-09T06:19:12,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152207) roll requested 2024-12-09T06:19:12,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152214 2024-12-09T06:19:12,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152214 2024-12-09T06:19:12,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152214) roll requested 2024-12-09T06:19:12,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152218 2024-12-09T06:19:12,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152218 2024-12-09T06:19:12,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152218) roll requested 2024-12-09T06:19:12,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152223 2024-12-09T06:19:12,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152223 2024-12-09T06:19:12,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152223) roll requested 2024-12-09T06:19:12,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152230 2024-12-09T06:19:12,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152230 2024-12-09T06:19:12,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152230) roll requested 2024-12-09T06:19:12,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152237 2024-12-09T06:19:12,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152237 2024-12-09T06:19:12,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152237) roll requested 2024-12-09T06:19:12,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152243 2024-12-09T06:19:12,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152243 2024-12-09T06:19:12,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152243) roll requested 2024-12-09T06:19:12,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152252 2024-12-09T06:19:12,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152252 2024-12-09T06:19:12,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152252) roll requested 2024-12-09T06:19:12,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152261 2024-12-09T06:19:12,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152261 2024-12-09T06:19:12,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152261) roll requested 2024-12-09T06:19:12,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152270 2024-12-09T06:19:12,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152270 2024-12-09T06:19:12,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152270) roll requested 2024-12-09T06:19:12,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152276 2024-12-09T06:19:12,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152276 2024-12-09T06:19:12,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152276) roll requested 2024-12-09T06:19:12,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152285 2024-12-09T06:19:12,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152285 2024-12-09T06:19:12,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152285) roll requested 2024-12-09T06:19:12,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152293 2024-12-09T06:19:12,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152293 2024-12-09T06:19:12,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152293) roll requested 2024-12-09T06:19:12,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152298 2024-12-09T06:19:12,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152298 2024-12-09T06:19:12,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152298) roll requested 2024-12-09T06:19:12,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152303 2024-12-09T06:19:12,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152303 2024-12-09T06:19:12,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152303) roll requested 2024-12-09T06:19:12,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152308 2024-12-09T06:19:12,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152308 2024-12-09T06:19:12,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152308) roll requested 2024-12-09T06:19:12,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152314 2024-12-09T06:19:12,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152314 2024-12-09T06:19:12,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152314) roll requested 2024-12-09T06:19:12,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152321 2024-12-09T06:19:12,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152321 2024-12-09T06:19:12,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152321) roll requested 2024-12-09T06:19:12,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152328 2024-12-09T06:19:12,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152328 2024-12-09T06:19:12,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152328) roll requested 2024-12-09T06:19:12,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152336 2024-12-09T06:19:12,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152336 2024-12-09T06:19:12,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152336) roll requested 2024-12-09T06:19:12,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152343 2024-12-09T06:19:12,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152343 2024-12-09T06:19:12,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152343) roll requested 2024-12-09T06:19:12,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152352 2024-12-09T06:19:12,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152352 2024-12-09T06:19:12,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152352) roll requested 2024-12-09T06:19:12,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152361 2024-12-09T06:19:12,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152361 2024-12-09T06:19:12,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152361) roll requested 2024-12-09T06:19:12,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152371 2024-12-09T06:19:12,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152371 2024-12-09T06:19:12,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152371) roll requested 2024-12-09T06:19:12,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152380 2024-12-09T06:19:12,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152380 2024-12-09T06:19:12,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152380) roll requested 2024-12-09T06:19:12,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152387 2024-12-09T06:19:12,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152387 2024-12-09T06:19:12,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152387) roll requested 2024-12-09T06:19:12,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152394 2024-12-09T06:19:12,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152394 2024-12-09T06:19:12,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152394) roll requested 2024-12-09T06:19:12,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152402 2024-12-09T06:19:12,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152402 2024-12-09T06:19:12,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152402) roll requested 2024-12-09T06:19:12,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152411 2024-12-09T06:19:12,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152411 2024-12-09T06:19:12,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152411) roll requested 2024-12-09T06:19:12,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152419 2024-12-09T06:19:12,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152419 2024-12-09T06:19:12,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152419) roll requested 2024-12-09T06:19:12,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152426 2024-12-09T06:19:12,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152426 2024-12-09T06:19:12,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152426) roll requested 2024-12-09T06:19:12,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152435 2024-12-09T06:19:12,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152435 2024-12-09T06:19:12,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152435) roll requested 2024-12-09T06:19:12,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152442 2024-12-09T06:19:12,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152442 2024-12-09T06:19:12,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152442) roll requested 2024-12-09T06:19:12,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152452 2024-12-09T06:19:12,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152452 2024-12-09T06:19:12,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152452) roll requested 2024-12-09T06:19:12,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152459 2024-12-09T06:19:12,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152459 2024-12-09T06:19:12,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152459) roll requested 2024-12-09T06:19:12,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152467 2024-12-09T06:19:12,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152467 2024-12-09T06:19:12,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152467) roll requested 2024-12-09T06:19:12,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152476 2024-12-09T06:19:12,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152476 2024-12-09T06:19:12,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152476) roll requested 2024-12-09T06:19:12,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152486 2024-12-09T06:19:12,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152486 2024-12-09T06:19:12,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152486) roll requested 2024-12-09T06:19:12,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152494 2024-12-09T06:19:12,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152494 2024-12-09T06:19:12,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152494) roll requested 2024-12-09T06:19:12,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152502 2024-12-09T06:19:12,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152502 2024-12-09T06:19:12,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152502) roll requested 2024-12-09T06:19:12,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152511 2024-12-09T06:19:12,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152511 2024-12-09T06:19:12,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152511) roll requested 2024-12-09T06:19:12,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152520 2024-12-09T06:19:12,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152520 2024-12-09T06:19:12,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152520) roll requested 2024-12-09T06:19:12,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152527 2024-12-09T06:19:12,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152527 2024-12-09T06:19:12,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152527) roll requested 2024-12-09T06:19:12,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152534 2024-12-09T06:19:12,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152534 2024-12-09T06:19:12,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152534) roll requested 2024-12-09T06:19:12,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152542 2024-12-09T06:19:12,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152542 2024-12-09T06:19:12,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152542) roll requested 2024-12-09T06:19:12,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152553 2024-12-09T06:19:12,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152553 2024-12-09T06:19:12,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152553) roll requested 2024-12-09T06:19:12,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152561 2024-12-09T06:19:12,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152561 2024-12-09T06:19:12,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152561) roll requested 2024-12-09T06:19:12,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152566 2024-12-09T06:19:12,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152566 2024-12-09T06:19:12,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152566) roll requested 2024-12-09T06:19:12,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152571 2024-12-09T06:19:12,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152571 2024-12-09T06:19:12,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152571) roll requested 2024-12-09T06:19:12,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152577 2024-12-09T06:19:12,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152577 2024-12-09T06:19:12,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152577) roll requested 2024-12-09T06:19:12,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152585 2024-12-09T06:19:12,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152585 2024-12-09T06:19:12,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152585) roll requested 2024-12-09T06:19:12,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152592 2024-12-09T06:19:12,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152592 2024-12-09T06:19:12,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152592) roll requested 2024-12-09T06:19:12,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152599 2024-12-09T06:19:12,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152599 2024-12-09T06:19:12,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152599) roll requested 2024-12-09T06:19:12,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152606 2024-12-09T06:19:12,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152606 2024-12-09T06:19:12,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152606) roll requested 2024-12-09T06:19:12,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152616 2024-12-09T06:19:12,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152616 2024-12-09T06:19:12,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152616) roll requested 2024-12-09T06:19:12,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152627 2024-12-09T06:19:12,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152627 2024-12-09T06:19:12,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152627) roll requested 2024-12-09T06:19:12,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152635 2024-12-09T06:19:12,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152635 2024-12-09T06:19:12,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152635) roll requested 2024-12-09T06:19:12,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152640 2024-12-09T06:19:12,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152640 2024-12-09T06:19:12,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152640) roll requested 2024-12-09T06:19:12,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152645 2024-12-09T06:19:12,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152645 2024-12-09T06:19:12,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152645) roll requested 2024-12-09T06:19:12,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152651 2024-12-09T06:19:12,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152651 2024-12-09T06:19:12,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152651) roll requested 2024-12-09T06:19:12,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152659 2024-12-09T06:19:12,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152659 2024-12-09T06:19:12,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152659) roll requested 2024-12-09T06:19:12,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152667 2024-12-09T06:19:12,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152667 2024-12-09T06:19:12,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152667) roll requested 2024-12-09T06:19:12,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152675 2024-12-09T06:19:12,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152675 2024-12-09T06:19:12,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152675) roll requested 2024-12-09T06:19:12,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152683 2024-12-09T06:19:12,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152683 2024-12-09T06:19:12,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152683) roll requested 2024-12-09T06:19:12,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152692 2024-12-09T06:19:12,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152692 2024-12-09T06:19:12,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152692) roll requested 2024-12-09T06:19:12,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152699 2024-12-09T06:19:12,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152699 2024-12-09T06:19:12,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152699) roll requested 2024-12-09T06:19:12,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152708 2024-12-09T06:19:12,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152708 2024-12-09T06:19:12,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152708) roll requested 2024-12-09T06:19:12,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152716 2024-12-09T06:19:12,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152716 2024-12-09T06:19:12,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152716) roll requested 2024-12-09T06:19:12,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152725 2024-12-09T06:19:12,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152725 2024-12-09T06:19:12,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152725) roll requested 2024-12-09T06:19:12,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152733 2024-12-09T06:19:12,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152733 2024-12-09T06:19:12,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152733) roll requested 2024-12-09T06:19:12,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152743 2024-12-09T06:19:12,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152743 2024-12-09T06:19:12,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152743) roll requested 2024-12-09T06:19:12,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152752 2024-12-09T06:19:12,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152752 2024-12-09T06:19:12,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152752) roll requested 2024-12-09T06:19:12,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152761 2024-12-09T06:19:12,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152761 2024-12-09T06:19:12,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152761) roll requested 2024-12-09T06:19:12,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152772 2024-12-09T06:19:12,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152772 2024-12-09T06:19:12,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152772) roll requested 2024-12-09T06:19:12,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152777 2024-12-09T06:19:12,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152777 2024-12-09T06:19:12,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152777) roll requested 2024-12-09T06:19:12,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152787 2024-12-09T06:19:12,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152787 2024-12-09T06:19:12,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152787) roll requested 2024-12-09T06:19:12,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152793 2024-12-09T06:19:12,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152793 2024-12-09T06:19:12,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152793) roll requested 2024-12-09T06:19:12,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152798 2024-12-09T06:19:12,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152798 2024-12-09T06:19:12,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152798) roll requested 2024-12-09T06:19:12,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152803 2024-12-09T06:19:12,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152803 2024-12-09T06:19:12,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152803) roll requested 2024-12-09T06:19:12,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152814 2024-12-09T06:19:12,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152814 2024-12-09T06:19:12,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152814) roll requested 2024-12-09T06:19:12,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152820 2024-12-09T06:19:12,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152820 2024-12-09T06:19:12,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152820) roll requested 2024-12-09T06:19:12,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152827 2024-12-09T06:19:12,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152827 2024-12-09T06:19:12,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152827) roll requested 2024-12-09T06:19:12,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152834 2024-12-09T06:19:12,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152834 2024-12-09T06:19:12,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152834) roll requested 2024-12-09T06:19:12,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152841 2024-12-09T06:19:12,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152841 2024-12-09T06:19:12,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152841) roll requested 2024-12-09T06:19:12,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152848 2024-12-09T06:19:12,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152848 2024-12-09T06:19:12,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152848) roll requested 2024-12-09T06:19:12,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152856 2024-12-09T06:19:12,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152856 2024-12-09T06:19:12,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152856) roll requested 2024-12-09T06:19:12,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152863 2024-12-09T06:19:12,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152863 2024-12-09T06:19:12,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152863) roll requested 2024-12-09T06:19:12,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152869 2024-12-09T06:19:12,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152869 2024-12-09T06:19:12,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152869) roll requested 2024-12-09T06:19:12,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152876 2024-12-09T06:19:12,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152876 2024-12-09T06:19:12,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152876) roll requested 2024-12-09T06:19:12,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152882 2024-12-09T06:19:12,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152882 2024-12-09T06:19:12,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152882) roll requested 2024-12-09T06:19:12,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152889 2024-12-09T06:19:12,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152889 2024-12-09T06:19:12,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152889) roll requested 2024-12-09T06:19:12,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152896 2024-12-09T06:19:12,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152896 2024-12-09T06:19:12,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152896) roll requested 2024-12-09T06:19:12,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152904 2024-12-09T06:19:12,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152904 2024-12-09T06:19:12,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152904) roll requested 2024-12-09T06:19:12,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152913 2024-12-09T06:19:12,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152913 2024-12-09T06:19:12,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152913) roll requested 2024-12-09T06:19:12,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152923 2024-12-09T06:19:12,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152923 2024-12-09T06:19:12,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152923) roll requested 2024-12-09T06:19:12,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152931 2024-12-09T06:19:12,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152931 2024-12-09T06:19:12,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152931) roll requested 2024-12-09T06:19:12,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152938 2024-12-09T06:19:12,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152938 2024-12-09T06:19:12,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152938) roll requested 2024-12-09T06:19:12,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152946 2024-12-09T06:19:12,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152946 2024-12-09T06:19:12,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152946) roll requested 2024-12-09T06:19:12,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152956 2024-12-09T06:19:12,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152956 2024-12-09T06:19:12,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152956) roll requested 2024-12-09T06:19:12,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152967 2024-12-09T06:19:12,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152967 2024-12-09T06:19:12,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152967) roll requested 2024-12-09T06:19:12,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152976 2024-12-09T06:19:12,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152976 2024-12-09T06:19:12,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152976) roll requested 2024-12-09T06:19:12,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152984 2024-12-09T06:19:12,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152984 2024-12-09T06:19:12,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152984) roll requested 2024-12-09T06:19:12,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152990 2024-12-09T06:19:12,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:12,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152990 2024-12-09T06:19:12,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:12,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:12,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152990) roll requested 2024-12-09T06:19:12,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725152999 2024-12-09T06:19:13,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152999 2024-12-09T06:19:13,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725152999) roll requested 2024-12-09T06:19:13,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153007 2024-12-09T06:19:13,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725152999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153007 2024-12-09T06:19:13,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153007) roll requested 2024-12-09T06:19:13,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153017 2024-12-09T06:19:13,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153017 2024-12-09T06:19:13,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153017) roll requested 2024-12-09T06:19:13,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153025 2024-12-09T06:19:13,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153025 2024-12-09T06:19:13,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153025) roll requested 2024-12-09T06:19:13,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153033 2024-12-09T06:19:13,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153033 2024-12-09T06:19:13,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153033) roll requested 2024-12-09T06:19:13,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153040 2024-12-09T06:19:13,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153040 2024-12-09T06:19:13,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153040) roll requested 2024-12-09T06:19:13,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153048 2024-12-09T06:19:13,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153048 2024-12-09T06:19:13,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153048) roll requested 2024-12-09T06:19:13,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153065 2024-12-09T06:19:13,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153065 2024-12-09T06:19:13,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153065) roll requested 2024-12-09T06:19:13,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153078 2024-12-09T06:19:13,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153078 2024-12-09T06:19:13,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153078) roll requested 2024-12-09T06:19:13,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153085 2024-12-09T06:19:13,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153085 2024-12-09T06:19:13,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153085) roll requested 2024-12-09T06:19:13,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153089 2024-12-09T06:19:13,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153089 2024-12-09T06:19:13,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153089) roll requested 2024-12-09T06:19:13,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153094 2024-12-09T06:19:13,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153094 2024-12-09T06:19:13,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153094) roll requested 2024-12-09T06:19:13,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153100 2024-12-09T06:19:13,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153100 2024-12-09T06:19:13,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153100) roll requested 2024-12-09T06:19:13,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153106 2024-12-09T06:19:13,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153106 2024-12-09T06:19:13,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153106) roll requested 2024-12-09T06:19:13,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153113 2024-12-09T06:19:13,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153113 2024-12-09T06:19:13,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153113) roll requested 2024-12-09T06:19:13,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153121 2024-12-09T06:19:13,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153121 2024-12-09T06:19:13,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153121) roll requested 2024-12-09T06:19:13,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153128 2024-12-09T06:19:13,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153128 2024-12-09T06:19:13,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153128) roll requested 2024-12-09T06:19:13,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153136 2024-12-09T06:19:13,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153136 2024-12-09T06:19:13,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153136) roll requested 2024-12-09T06:19:13,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153146 2024-12-09T06:19:13,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153146 2024-12-09T06:19:13,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153146) roll requested 2024-12-09T06:19:13,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153155 2024-12-09T06:19:13,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153155 2024-12-09T06:19:13,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153155) roll requested 2024-12-09T06:19:13,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153165 2024-12-09T06:19:13,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153165 2024-12-09T06:19:13,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153165) roll requested 2024-12-09T06:19:13,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153170 2024-12-09T06:19:13,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153170 2024-12-09T06:19:13,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153170) roll requested 2024-12-09T06:19:13,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153175 2024-12-09T06:19:13,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153175 2024-12-09T06:19:13,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153175) roll requested 2024-12-09T06:19:13,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153179 2024-12-09T06:19:13,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153179 2024-12-09T06:19:13,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153179) roll requested 2024-12-09T06:19:13,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153184 2024-12-09T06:19:13,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153184 2024-12-09T06:19:13,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153184) roll requested 2024-12-09T06:19:13,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153191 2024-12-09T06:19:13,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153191 2024-12-09T06:19:13,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153191) roll requested 2024-12-09T06:19:13,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153198 2024-12-09T06:19:13,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153198 2024-12-09T06:19:13,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153198) roll requested 2024-12-09T06:19:13,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153210 2024-12-09T06:19:13,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153210 2024-12-09T06:19:13,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153210) roll requested 2024-12-09T06:19:13,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153217 2024-12-09T06:19:13,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153217 2024-12-09T06:19:13,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153217) roll requested 2024-12-09T06:19:13,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153226 2024-12-09T06:19:13,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153226 2024-12-09T06:19:13,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153226) roll requested 2024-12-09T06:19:13,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153232 2024-12-09T06:19:13,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153232 2024-12-09T06:19:13,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153232) roll requested 2024-12-09T06:19:13,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153242 2024-12-09T06:19:13,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153242 2024-12-09T06:19:13,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153242) roll requested 2024-12-09T06:19:13,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153255 2024-12-09T06:19:13,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153255 2024-12-09T06:19:13,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153255) roll requested 2024-12-09T06:19:13,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153261 2024-12-09T06:19:13,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153261 2024-12-09T06:19:13,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153261) roll requested 2024-12-09T06:19:13,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153267 2024-12-09T06:19:13,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153267 2024-12-09T06:19:13,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153267) roll requested 2024-12-09T06:19:13,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153274 2024-12-09T06:19:13,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153274 2024-12-09T06:19:13,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153274) roll requested 2024-12-09T06:19:13,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153281 2024-12-09T06:19:13,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153281 2024-12-09T06:19:13,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153281) roll requested 2024-12-09T06:19:13,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153293 2024-12-09T06:19:13,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153293 2024-12-09T06:19:13,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153293) roll requested 2024-12-09T06:19:13,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153305 2024-12-09T06:19:13,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153305 2024-12-09T06:19:13,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153305) roll requested 2024-12-09T06:19:13,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153322 2024-12-09T06:19:13,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153322 2024-12-09T06:19:13,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153322) roll requested 2024-12-09T06:19:13,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153337 2024-12-09T06:19:13,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153337 2024-12-09T06:19:13,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153337) roll requested 2024-12-09T06:19:13,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153343 2024-12-09T06:19:13,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153343 2024-12-09T06:19:13,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153343) roll requested 2024-12-09T06:19:13,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153352 2024-12-09T06:19:13,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153352 2024-12-09T06:19:13,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153352) roll requested 2024-12-09T06:19:13,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153357 2024-12-09T06:19:13,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153357 2024-12-09T06:19:13,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153357) roll requested 2024-12-09T06:19:13,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153362 2024-12-09T06:19:13,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153362 2024-12-09T06:19:13,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153362) roll requested 2024-12-09T06:19:13,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153368 2024-12-09T06:19:13,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153368 2024-12-09T06:19:13,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153368) roll requested 2024-12-09T06:19:13,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153374 2024-12-09T06:19:13,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153374 2024-12-09T06:19:13,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153374) roll requested 2024-12-09T06:19:13,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153379 2024-12-09T06:19:13,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153379 2024-12-09T06:19:13,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153379) roll requested 2024-12-09T06:19:13,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153384 2024-12-09T06:19:13,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153384 2024-12-09T06:19:13,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153384) roll requested 2024-12-09T06:19:13,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153390 2024-12-09T06:19:13,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153390 2024-12-09T06:19:13,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153390) roll requested 2024-12-09T06:19:13,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153395 2024-12-09T06:19:13,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153395 2024-12-09T06:19:13,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153395) roll requested 2024-12-09T06:19:13,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153400 2024-12-09T06:19:13,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153400 2024-12-09T06:19:13,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153400) roll requested 2024-12-09T06:19:13,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153409 2024-12-09T06:19:13,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153409 2024-12-09T06:19:13,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153409) roll requested 2024-12-09T06:19:13,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153418 2024-12-09T06:19:13,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153418 2024-12-09T06:19:13,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153418) roll requested 2024-12-09T06:19:13,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153424 2024-12-09T06:19:13,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153424 2024-12-09T06:19:13,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153424) roll requested 2024-12-09T06:19:13,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153429 2024-12-09T06:19:13,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153429 2024-12-09T06:19:13,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153429) roll requested 2024-12-09T06:19:13,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153434 2024-12-09T06:19:13,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153434 2024-12-09T06:19:13,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153434) roll requested 2024-12-09T06:19:13,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153440 2024-12-09T06:19:13,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153440 2024-12-09T06:19:13,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153440) roll requested 2024-12-09T06:19:13,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153447 2024-12-09T06:19:13,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153447 2024-12-09T06:19:13,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153447) roll requested 2024-12-09T06:19:13,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153456 2024-12-09T06:19:13,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153456 2024-12-09T06:19:13,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153456) roll requested 2024-12-09T06:19:13,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153461 2024-12-09T06:19:13,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153461 2024-12-09T06:19:13,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153461) roll requested 2024-12-09T06:19:13,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153466 2024-12-09T06:19:13,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153466 2024-12-09T06:19:13,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153466) roll requested 2024-12-09T06:19:13,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153472 2024-12-09T06:19:13,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153472 2024-12-09T06:19:13,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153472) roll requested 2024-12-09T06:19:13,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153478 2024-12-09T06:19:13,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153478 2024-12-09T06:19:13,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153478) roll requested 2024-12-09T06:19:13,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153485 2024-12-09T06:19:13,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153485 2024-12-09T06:19:13,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153485) roll requested 2024-12-09T06:19:13,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153493 2024-12-09T06:19:13,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153493 2024-12-09T06:19:13,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153493) roll requested 2024-12-09T06:19:13,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153500 2024-12-09T06:19:13,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153500 2024-12-09T06:19:13,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153500) roll requested 2024-12-09T06:19:13,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153507 2024-12-09T06:19:13,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153507 2024-12-09T06:19:13,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153507) roll requested 2024-12-09T06:19:13,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153515 2024-12-09T06:19:13,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153515 2024-12-09T06:19:13,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153515) roll requested 2024-12-09T06:19:13,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153523 2024-12-09T06:19:13,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153523 2024-12-09T06:19:13,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153523) roll requested 2024-12-09T06:19:13,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153532 2024-12-09T06:19:13,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153532 2024-12-09T06:19:13,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153532) roll requested 2024-12-09T06:19:13,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153539 2024-12-09T06:19:13,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153539 2024-12-09T06:19:13,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153539) roll requested 2024-12-09T06:19:13,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153547 2024-12-09T06:19:13,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153547 2024-12-09T06:19:13,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153547) roll requested 2024-12-09T06:19:13,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153556 2024-12-09T06:19:13,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153556 2024-12-09T06:19:13,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153556) roll requested 2024-12-09T06:19:13,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153565 2024-12-09T06:19:13,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153565 2024-12-09T06:19:13,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153565) roll requested 2024-12-09T06:19:13,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153572 2024-12-09T06:19:13,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153572 2024-12-09T06:19:13,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153572) roll requested 2024-12-09T06:19:13,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153585 2024-12-09T06:19:13,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153585 2024-12-09T06:19:13,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153585) roll requested 2024-12-09T06:19:13,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153594 2024-12-09T06:19:13,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153594 2024-12-09T06:19:13,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153594) roll requested 2024-12-09T06:19:13,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153601 2024-12-09T06:19:13,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153601 2024-12-09T06:19:13,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153601) roll requested 2024-12-09T06:19:13,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153612 2024-12-09T06:19:13,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153612 2024-12-09T06:19:13,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153612) roll requested 2024-12-09T06:19:13,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153621 2024-12-09T06:19:13,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153621 2024-12-09T06:19:13,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153621) roll requested 2024-12-09T06:19:13,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153627 2024-12-09T06:19:13,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153627 2024-12-09T06:19:13,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153627) roll requested 2024-12-09T06:19:13,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153634 2024-12-09T06:19:13,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153634 2024-12-09T06:19:13,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153634) roll requested 2024-12-09T06:19:13,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153641 2024-12-09T06:19:13,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153641 2024-12-09T06:19:13,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153641) roll requested 2024-12-09T06:19:13,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153647 2024-12-09T06:19:13,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153647 2024-12-09T06:19:13,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153647) roll requested 2024-12-09T06:19:13,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153656 2024-12-09T06:19:13,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153656 2024-12-09T06:19:13,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153656) roll requested 2024-12-09T06:19:13,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153662 2024-12-09T06:19:13,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153662 2024-12-09T06:19:13,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153662) roll requested 2024-12-09T06:19:13,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153667 2024-12-09T06:19:13,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153667 2024-12-09T06:19:13,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153667) roll requested 2024-12-09T06:19:13,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153676 2024-12-09T06:19:13,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153676 2024-12-09T06:19:13,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153676) roll requested 2024-12-09T06:19:13,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153682 2024-12-09T06:19:13,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153682 2024-12-09T06:19:13,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153682) roll requested 2024-12-09T06:19:13,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153688 2024-12-09T06:19:13,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153688 2024-12-09T06:19:13,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153688) roll requested 2024-12-09T06:19:13,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153693 2024-12-09T06:19:13,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153693 2024-12-09T06:19:13,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153693) roll requested 2024-12-09T06:19:13,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153699 2024-12-09T06:19:13,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153699 2024-12-09T06:19:13,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153699) roll requested 2024-12-09T06:19:13,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153703 2024-12-09T06:19:13,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153703 2024-12-09T06:19:13,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153703) roll requested 2024-12-09T06:19:13,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153709 2024-12-09T06:19:13,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153709 2024-12-09T06:19:13,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153709) roll requested 2024-12-09T06:19:13,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153715 2024-12-09T06:19:13,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153715 2024-12-09T06:19:13,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153715) roll requested 2024-12-09T06:19:13,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153722 2024-12-09T06:19:13,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153722 2024-12-09T06:19:13,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153722) roll requested 2024-12-09T06:19:13,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153727 2024-12-09T06:19:13,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153727 2024-12-09T06:19:13,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153727) roll requested 2024-12-09T06:19:13,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153733 2024-12-09T06:19:13,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153733 2024-12-09T06:19:13,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153733) roll requested 2024-12-09T06:19:13,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153738 2024-12-09T06:19:13,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153738 2024-12-09T06:19:13,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153738) roll requested 2024-12-09T06:19:13,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153744 2024-12-09T06:19:13,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153744 2024-12-09T06:19:13,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153744) roll requested 2024-12-09T06:19:13,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153750 2024-12-09T06:19:13,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153750 2024-12-09T06:19:13,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153750) roll requested 2024-12-09T06:19:13,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153755 2024-12-09T06:19:13,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153755 2024-12-09T06:19:13,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153755) roll requested 2024-12-09T06:19:13,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153760 2024-12-09T06:19:13,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153760 2024-12-09T06:19:13,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153760) roll requested 2024-12-09T06:19:13,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153768 2024-12-09T06:19:13,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153768 2024-12-09T06:19:13,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153768) roll requested 2024-12-09T06:19:13,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153774 2024-12-09T06:19:13,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153774 2024-12-09T06:19:13,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153774) roll requested 2024-12-09T06:19:13,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153779 2024-12-09T06:19:13,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153779 2024-12-09T06:19:13,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153779) roll requested 2024-12-09T06:19:13,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153784 2024-12-09T06:19:13,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153784 2024-12-09T06:19:13,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153784) roll requested 2024-12-09T06:19:13,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153790 2024-12-09T06:19:13,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153790 2024-12-09T06:19:13,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153790) roll requested 2024-12-09T06:19:13,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153797 2024-12-09T06:19:13,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153797 2024-12-09T06:19:13,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153797) roll requested 2024-12-09T06:19:13,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153805 2024-12-09T06:19:13,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153805 2024-12-09T06:19:13,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153805) roll requested 2024-12-09T06:19:13,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153812 2024-12-09T06:19:13,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153812 2024-12-09T06:19:13,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153812) roll requested 2024-12-09T06:19:13,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153820 2024-12-09T06:19:13,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153820 2024-12-09T06:19:13,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153820) roll requested 2024-12-09T06:19:13,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153828 2024-12-09T06:19:13,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153828 2024-12-09T06:19:13,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153828) roll requested 2024-12-09T06:19:13,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153835 2024-12-09T06:19:13,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153835 2024-12-09T06:19:13,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153835) roll requested 2024-12-09T06:19:13,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153843 2024-12-09T06:19:13,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153843 2024-12-09T06:19:13,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153843) roll requested 2024-12-09T06:19:13,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153852 2024-12-09T06:19:13,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153852 2024-12-09T06:19:13,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153852) roll requested 2024-12-09T06:19:13,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153861 2024-12-09T06:19:13,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153861 2024-12-09T06:19:13,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153861) roll requested 2024-12-09T06:19:13,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153869 2024-12-09T06:19:13,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153869 2024-12-09T06:19:13,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153869) roll requested 2024-12-09T06:19:13,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153875 2024-12-09T06:19:13,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153875 2024-12-09T06:19:13,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153875) roll requested 2024-12-09T06:19:13,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153881 2024-12-09T06:19:13,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153881 2024-12-09T06:19:13,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153881) roll requested 2024-12-09T06:19:13,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153889 2024-12-09T06:19:13,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153889 2024-12-09T06:19:13,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153889) roll requested 2024-12-09T06:19:13,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153896 2024-12-09T06:19:13,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153896 2024-12-09T06:19:13,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153896) roll requested 2024-12-09T06:19:13,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153903 2024-12-09T06:19:13,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153903 2024-12-09T06:19:13,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153903) roll requested 2024-12-09T06:19:13,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153913 2024-12-09T06:19:13,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153913 2024-12-09T06:19:13,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153913) roll requested 2024-12-09T06:19:13,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153921 2024-12-09T06:19:13,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153921 2024-12-09T06:19:13,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153921) roll requested 2024-12-09T06:19:13,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153930 2024-12-09T06:19:13,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153930 2024-12-09T06:19:13,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153930) roll requested 2024-12-09T06:19:13,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153939 2024-12-09T06:19:13,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153939 2024-12-09T06:19:13,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153939) roll requested 2024-12-09T06:19:13,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153948 2024-12-09T06:19:13,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153948 2024-12-09T06:19:13,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153948) roll requested 2024-12-09T06:19:13,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153957 2024-12-09T06:19:13,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153957 2024-12-09T06:19:13,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153957) roll requested 2024-12-09T06:19:13,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153965 2024-12-09T06:19:13,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153965 2024-12-09T06:19:13,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153965) roll requested 2024-12-09T06:19:13,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153971 2024-12-09T06:19:13,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153971 2024-12-09T06:19:13,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153971) roll requested 2024-12-09T06:19:13,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153979 2024-12-09T06:19:13,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153979 2024-12-09T06:19:13,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153979) roll requested 2024-12-09T06:19:13,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153985 2024-12-09T06:19:13,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153985 2024-12-09T06:19:13,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153985) roll requested 2024-12-09T06:19:13,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153992 2024-12-09T06:19:13,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:13,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153992 2024-12-09T06:19:13,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:13,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:13,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153992) roll requested 2024-12-09T06:19:13,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725153998 2024-12-09T06:19:14,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153998 2024-12-09T06:19:14,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725153998) roll requested 2024-12-09T06:19:14,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154005 2024-12-09T06:19:14,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725153998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154005 2024-12-09T06:19:14,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154005) roll requested 2024-12-09T06:19:14,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154013 2024-12-09T06:19:14,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154013 2024-12-09T06:19:14,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154013) roll requested 2024-12-09T06:19:14,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154021 2024-12-09T06:19:14,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154021 2024-12-09T06:19:14,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154021) roll requested 2024-12-09T06:19:14,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154028 2024-12-09T06:19:14,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154028 2024-12-09T06:19:14,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154028) roll requested 2024-12-09T06:19:14,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154036 2024-12-09T06:19:14,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154036 2024-12-09T06:19:14,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154036) roll requested 2024-12-09T06:19:14,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154045 2024-12-09T06:19:14,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154045 2024-12-09T06:19:14,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154045) roll requested 2024-12-09T06:19:14,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154054 2024-12-09T06:19:14,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154054 2024-12-09T06:19:14,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154054) roll requested 2024-12-09T06:19:14,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154062 2024-12-09T06:19:14,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154062 2024-12-09T06:19:14,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154062) roll requested 2024-12-09T06:19:14,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154069 2024-12-09T06:19:14,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154069 2024-12-09T06:19:14,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154069) roll requested 2024-12-09T06:19:14,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154077 2024-12-09T06:19:14,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154077 2024-12-09T06:19:14,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154077) roll requested 2024-12-09T06:19:14,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154083 2024-12-09T06:19:14,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154083 2024-12-09T06:19:14,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154083) roll requested 2024-12-09T06:19:14,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154089 2024-12-09T06:19:14,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154089 2024-12-09T06:19:14,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154089) roll requested 2024-12-09T06:19:14,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154094 2024-12-09T06:19:14,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154094 2024-12-09T06:19:14,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154094) roll requested 2024-12-09T06:19:14,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154099 2024-12-09T06:19:14,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154099 2024-12-09T06:19:14,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154099) roll requested 2024-12-09T06:19:14,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154104 2024-12-09T06:19:14,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154104 2024-12-09T06:19:14,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154104) roll requested 2024-12-09T06:19:14,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154109 2024-12-09T06:19:14,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154109 2024-12-09T06:19:14,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154109) roll requested 2024-12-09T06:19:14,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154115 2024-12-09T06:19:14,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154115 2024-12-09T06:19:14,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154115) roll requested 2024-12-09T06:19:14,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154125 2024-12-09T06:19:14,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154125 2024-12-09T06:19:14,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154125) roll requested 2024-12-09T06:19:14,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154132 2024-12-09T06:19:14,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154132 2024-12-09T06:19:14,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154132) roll requested 2024-12-09T06:19:14,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154137 2024-12-09T06:19:14,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154137 2024-12-09T06:19:14,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154137) roll requested 2024-12-09T06:19:14,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154142 2024-12-09T06:19:14,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154142 2024-12-09T06:19:14,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154142) roll requested 2024-12-09T06:19:14,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154147 2024-12-09T06:19:14,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154147 2024-12-09T06:19:14,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154147) roll requested 2024-12-09T06:19:14,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154152 2024-12-09T06:19:14,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154152 2024-12-09T06:19:14,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154152) roll requested 2024-12-09T06:19:14,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154156 2024-12-09T06:19:14,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154156 2024-12-09T06:19:14,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154156) roll requested 2024-12-09T06:19:14,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154164 2024-12-09T06:19:14,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154164 2024-12-09T06:19:14,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154164) roll requested 2024-12-09T06:19:14,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154170 2024-12-09T06:19:14,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154170 2024-12-09T06:19:14,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154170) roll requested 2024-12-09T06:19:14,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154178 2024-12-09T06:19:14,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154178 2024-12-09T06:19:14,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154178) roll requested 2024-12-09T06:19:14,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154187 2024-12-09T06:19:14,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154187 2024-12-09T06:19:14,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154187) roll requested 2024-12-09T06:19:14,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154195 2024-12-09T06:19:14,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154195 2024-12-09T06:19:14,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154195) roll requested 2024-12-09T06:19:14,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154200 2024-12-09T06:19:14,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154200 2024-12-09T06:19:14,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154200) roll requested 2024-12-09T06:19:14,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154205 2024-12-09T06:19:14,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154205 2024-12-09T06:19:14,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154205) roll requested 2024-12-09T06:19:14,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154210 2024-12-09T06:19:14,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154210 2024-12-09T06:19:14,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154210) roll requested 2024-12-09T06:19:14,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154217 2024-12-09T06:19:14,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154217 2024-12-09T06:19:14,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154217) roll requested 2024-12-09T06:19:14,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154222 2024-12-09T06:19:14,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154222 2024-12-09T06:19:14,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154222) roll requested 2024-12-09T06:19:14,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154232 2024-12-09T06:19:14,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154232 2024-12-09T06:19:14,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154232) roll requested 2024-12-09T06:19:14,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154237 2024-12-09T06:19:14,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154237 2024-12-09T06:19:14,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154237) roll requested 2024-12-09T06:19:14,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154242 2024-12-09T06:19:14,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154242 2024-12-09T06:19:14,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154242) roll requested 2024-12-09T06:19:14,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154247 2024-12-09T06:19:14,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154247 2024-12-09T06:19:14,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154247) roll requested 2024-12-09T06:19:14,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154253 2024-12-09T06:19:14,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154253 2024-12-09T06:19:14,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154253) roll requested 2024-12-09T06:19:14,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154258 2024-12-09T06:19:14,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154258 2024-12-09T06:19:14,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154258) roll requested 2024-12-09T06:19:14,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154263 2024-12-09T06:19:14,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154263 2024-12-09T06:19:14,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154263) roll requested 2024-12-09T06:19:14,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154273 2024-12-09T06:19:14,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154273 2024-12-09T06:19:14,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154273) roll requested 2024-12-09T06:19:14,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154279 2024-12-09T06:19:14,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154279 2024-12-09T06:19:14,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154279) roll requested 2024-12-09T06:19:14,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154285 2024-12-09T06:19:14,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154285 2024-12-09T06:19:14,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154285) roll requested 2024-12-09T06:19:14,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154293 2024-12-09T06:19:14,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154293 2024-12-09T06:19:14,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154293) roll requested 2024-12-09T06:19:14,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154301 2024-12-09T06:19:14,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154301 2024-12-09T06:19:14,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154301) roll requested 2024-12-09T06:19:14,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154306 2024-12-09T06:19:14,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154306 2024-12-09T06:19:14,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154306) roll requested 2024-12-09T06:19:14,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154311 2024-12-09T06:19:14,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154311 2024-12-09T06:19:14,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154311) roll requested 2024-12-09T06:19:14,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154317 2024-12-09T06:19:14,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154317 2024-12-09T06:19:14,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154317) roll requested 2024-12-09T06:19:14,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154326 2024-12-09T06:19:14,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154326 2024-12-09T06:19:14,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154326) roll requested 2024-12-09T06:19:14,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154334 2024-12-09T06:19:14,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154334 2024-12-09T06:19:14,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154334) roll requested 2024-12-09T06:19:14,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154344 2024-12-09T06:19:14,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154344 2024-12-09T06:19:14,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154344) roll requested 2024-12-09T06:19:14,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154353 2024-12-09T06:19:14,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154353 2024-12-09T06:19:14,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154353) roll requested 2024-12-09T06:19:14,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154362 2024-12-09T06:19:14,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154362 2024-12-09T06:19:14,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154362) roll requested 2024-12-09T06:19:14,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154371 2024-12-09T06:19:14,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154371 2024-12-09T06:19:14,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154371) roll requested 2024-12-09T06:19:14,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154377 2024-12-09T06:19:14,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154377 2024-12-09T06:19:14,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154377) roll requested 2024-12-09T06:19:14,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154386 2024-12-09T06:19:14,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154386 2024-12-09T06:19:14,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154386) roll requested 2024-12-09T06:19:14,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154393 2024-12-09T06:19:14,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154393 2024-12-09T06:19:14,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154393) roll requested 2024-12-09T06:19:14,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154398 2024-12-09T06:19:14,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154398 2024-12-09T06:19:14,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154398) roll requested 2024-12-09T06:19:14,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154404 2024-12-09T06:19:14,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154404 2024-12-09T06:19:14,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154404) roll requested 2024-12-09T06:19:14,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154411 2024-12-09T06:19:14,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154411 2024-12-09T06:19:14,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154411) roll requested 2024-12-09T06:19:14,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154418 2024-12-09T06:19:14,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154418 2024-12-09T06:19:14,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154418) roll requested 2024-12-09T06:19:14,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154424 2024-12-09T06:19:14,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154424 2024-12-09T06:19:14,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154424) roll requested 2024-12-09T06:19:14,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154431 2024-12-09T06:19:14,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154431 2024-12-09T06:19:14,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154431) roll requested 2024-12-09T06:19:14,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154440 2024-12-09T06:19:14,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154440 2024-12-09T06:19:14,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154440) roll requested 2024-12-09T06:19:14,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154449 2024-12-09T06:19:14,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154449 2024-12-09T06:19:14,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154449) roll requested 2024-12-09T06:19:14,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154456 2024-12-09T06:19:14,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154456 2024-12-09T06:19:14,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154456) roll requested 2024-12-09T06:19:14,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154464 2024-12-09T06:19:14,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154464 2024-12-09T06:19:14,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154464) roll requested 2024-12-09T06:19:14,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154472 2024-12-09T06:19:14,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154472 2024-12-09T06:19:14,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154472) roll requested 2024-12-09T06:19:14,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154476 2024-12-09T06:19:14,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154476 2024-12-09T06:19:14,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154476) roll requested 2024-12-09T06:19:14,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154481 2024-12-09T06:19:14,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154481 2024-12-09T06:19:14,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154481) roll requested 2024-12-09T06:19:14,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154489 2024-12-09T06:19:14,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154489 2024-12-09T06:19:14,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154489) roll requested 2024-12-09T06:19:14,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154496 2024-12-09T06:19:14,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154496 2024-12-09T06:19:14,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154496) roll requested 2024-12-09T06:19:14,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154502 2024-12-09T06:19:14,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154502 2024-12-09T06:19:14,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154502) roll requested 2024-12-09T06:19:14,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154511 2024-12-09T06:19:14,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154511 2024-12-09T06:19:14,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154511) roll requested 2024-12-09T06:19:14,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154518 2024-12-09T06:19:14,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154518 2024-12-09T06:19:14,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154518) roll requested 2024-12-09T06:19:14,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154531 2024-12-09T06:19:14,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154531 2024-12-09T06:19:14,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154531) roll requested 2024-12-09T06:19:14,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154539 2024-12-09T06:19:14,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154539 2024-12-09T06:19:14,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154539) roll requested 2024-12-09T06:19:14,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154543 2024-12-09T06:19:14,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154543 2024-12-09T06:19:14,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154543) roll requested 2024-12-09T06:19:14,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154548 2024-12-09T06:19:14,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154548 2024-12-09T06:19:14,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154548) roll requested 2024-12-09T06:19:14,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154557 2024-12-09T06:19:14,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154557 2024-12-09T06:19:14,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154557) roll requested 2024-12-09T06:19:14,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154563 2024-12-09T06:19:14,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154563 2024-12-09T06:19:14,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154563) roll requested 2024-12-09T06:19:14,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154568 2024-12-09T06:19:14,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154568 2024-12-09T06:19:14,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154568) roll requested 2024-12-09T06:19:14,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154575 2024-12-09T06:19:14,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154575 2024-12-09T06:19:14,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154575) roll requested 2024-12-09T06:19:14,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154580 2024-12-09T06:19:14,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154580 2024-12-09T06:19:14,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154580) roll requested 2024-12-09T06:19:14,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154587 2024-12-09T06:19:14,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154587 2024-12-09T06:19:14,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154587) roll requested 2024-12-09T06:19:14,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154595 2024-12-09T06:19:14,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154595 2024-12-09T06:19:14,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154595) roll requested 2024-12-09T06:19:14,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154601 2024-12-09T06:19:14,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154601 2024-12-09T06:19:14,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154601) roll requested 2024-12-09T06:19:14,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154605 2024-12-09T06:19:14,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154605 2024-12-09T06:19:14,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154605) roll requested 2024-12-09T06:19:14,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154611 2024-12-09T06:19:14,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154611 2024-12-09T06:19:14,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154611) roll requested 2024-12-09T06:19:14,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154616 2024-12-09T06:19:14,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154616 2024-12-09T06:19:14,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154616) roll requested 2024-12-09T06:19:14,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154623 2024-12-09T06:19:14,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154623 2024-12-09T06:19:14,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154623) roll requested 2024-12-09T06:19:14,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154630 2024-12-09T06:19:14,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154630 2024-12-09T06:19:14,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154630) roll requested 2024-12-09T06:19:14,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154637 2024-12-09T06:19:14,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154637 2024-12-09T06:19:14,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154637) roll requested 2024-12-09T06:19:14,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154646 2024-12-09T06:19:14,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154646 2024-12-09T06:19:14,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154646) roll requested 2024-12-09T06:19:14,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154656 2024-12-09T06:19:14,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154656 2024-12-09T06:19:14,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154656) roll requested 2024-12-09T06:19:14,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154664 2024-12-09T06:19:14,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154664 2024-12-09T06:19:14,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154664) roll requested 2024-12-09T06:19:14,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154673 2024-12-09T06:19:14,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154673 2024-12-09T06:19:14,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154673) roll requested 2024-12-09T06:19:14,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154680 2024-12-09T06:19:14,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154680 2024-12-09T06:19:14,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154680) roll requested 2024-12-09T06:19:14,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154687 2024-12-09T06:19:14,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154687 2024-12-09T06:19:14,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154687) roll requested 2024-12-09T06:19:14,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154695 2024-12-09T06:19:14,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154695 2024-12-09T06:19:14,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154695) roll requested 2024-12-09T06:19:14,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154699 2024-12-09T06:19:14,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154699 2024-12-09T06:19:14,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154699) roll requested 2024-12-09T06:19:14,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154704 2024-12-09T06:19:14,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154704 2024-12-09T06:19:14,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154704) roll requested 2024-12-09T06:19:14,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154709 2024-12-09T06:19:14,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154709 2024-12-09T06:19:14,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154709) roll requested 2024-12-09T06:19:14,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154716 2024-12-09T06:19:14,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154716 2024-12-09T06:19:14,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154716) roll requested 2024-12-09T06:19:14,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154722 2024-12-09T06:19:14,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154722 2024-12-09T06:19:14,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154722) roll requested 2024-12-09T06:19:14,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154729 2024-12-09T06:19:14,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154729 2024-12-09T06:19:14,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154729) roll requested 2024-12-09T06:19:14,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154736 2024-12-09T06:19:14,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154736 2024-12-09T06:19:14,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154736) roll requested 2024-12-09T06:19:14,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154744 2024-12-09T06:19:14,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154744 2024-12-09T06:19:14,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154744) roll requested 2024-12-09T06:19:14,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154752 2024-12-09T06:19:14,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154752 2024-12-09T06:19:14,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154752) roll requested 2024-12-09T06:19:14,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154760 2024-12-09T06:19:14,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154760 2024-12-09T06:19:14,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154760) roll requested 2024-12-09T06:19:14,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154768 2024-12-09T06:19:14,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154768 2024-12-09T06:19:14,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154768) roll requested 2024-12-09T06:19:14,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154776 2024-12-09T06:19:14,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154776 2024-12-09T06:19:14,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154776) roll requested 2024-12-09T06:19:14,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154783 2024-12-09T06:19:14,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154783 2024-12-09T06:19:14,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154783) roll requested 2024-12-09T06:19:14,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154790 2024-12-09T06:19:14,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154790 2024-12-09T06:19:14,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154790) roll requested 2024-12-09T06:19:14,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154797 2024-12-09T06:19:14,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154797 2024-12-09T06:19:14,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154797) roll requested 2024-12-09T06:19:14,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154804 2024-12-09T06:19:14,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154804 2024-12-09T06:19:14,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154804) roll requested 2024-12-09T06:19:14,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154810 2024-12-09T06:19:14,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154810 2024-12-09T06:19:14,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154810) roll requested 2024-12-09T06:19:14,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154818 2024-12-09T06:19:14,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154818 2024-12-09T06:19:14,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154818) roll requested 2024-12-09T06:19:14,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154827 2024-12-09T06:19:14,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154827 2024-12-09T06:19:14,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154827) roll requested 2024-12-09T06:19:14,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154837 2024-12-09T06:19:14,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154837 2024-12-09T06:19:14,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154837) roll requested 2024-12-09T06:19:14,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154845 2024-12-09T06:19:14,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154845 2024-12-09T06:19:14,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154845) roll requested 2024-12-09T06:19:14,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154854 2024-12-09T06:19:14,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154854 2024-12-09T06:19:14,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154854) roll requested 2024-12-09T06:19:14,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154862 2024-12-09T06:19:14,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154862 2024-12-09T06:19:14,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154862) roll requested 2024-12-09T06:19:14,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154873 2024-12-09T06:19:14,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154873 2024-12-09T06:19:14,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154873) roll requested 2024-12-09T06:19:14,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154880 2024-12-09T06:19:14,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154880 2024-12-09T06:19:14,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154880) roll requested 2024-12-09T06:19:14,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154886 2024-12-09T06:19:14,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154886 2024-12-09T06:19:14,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154886) roll requested 2024-12-09T06:19:14,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154895 2024-12-09T06:19:14,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154895 2024-12-09T06:19:14,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154895) roll requested 2024-12-09T06:19:14,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154906 2024-12-09T06:19:14,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154906 2024-12-09T06:19:14,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154906) roll requested 2024-12-09T06:19:14,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154914 2024-12-09T06:19:14,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154914 2024-12-09T06:19:14,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154914) roll requested 2024-12-09T06:19:14,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154920 2024-12-09T06:19:14,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154920 2024-12-09T06:19:14,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154920) roll requested 2024-12-09T06:19:14,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154925 2024-12-09T06:19:14,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154925 2024-12-09T06:19:14,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154925) roll requested 2024-12-09T06:19:14,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154930 2024-12-09T06:19:14,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154930 2024-12-09T06:19:14,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154930) roll requested 2024-12-09T06:19:14,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154936 2024-12-09T06:19:14,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154936 2024-12-09T06:19:14,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154936) roll requested 2024-12-09T06:19:14,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154942 2024-12-09T06:19:14,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154942 2024-12-09T06:19:14,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154942) roll requested 2024-12-09T06:19:14,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154947 2024-12-09T06:19:14,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154947 2024-12-09T06:19:14,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154947) roll requested 2024-12-09T06:19:14,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154953 2024-12-09T06:19:14,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154953 2024-12-09T06:19:14,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154953) roll requested 2024-12-09T06:19:14,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154959 2024-12-09T06:19:14,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154959 2024-12-09T06:19:14,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154959) roll requested 2024-12-09T06:19:14,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154964 2024-12-09T06:19:14,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154964 2024-12-09T06:19:14,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154964) roll requested 2024-12-09T06:19:14,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154970 2024-12-09T06:19:14,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154970 2024-12-09T06:19:14,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154970) roll requested 2024-12-09T06:19:14,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154976 2024-12-09T06:19:14,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154976 2024-12-09T06:19:14,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154976) roll requested 2024-12-09T06:19:14,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154981 2024-12-09T06:19:14,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154981 2024-12-09T06:19:14,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154981) roll requested 2024-12-09T06:19:14,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154987 2024-12-09T06:19:14,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154987 2024-12-09T06:19:14,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:14,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154987) roll requested 2024-12-09T06:19:14,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725154993 2024-12-09T06:19:14,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:14,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154993 2024-12-09T06:19:14,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:14,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725154993) roll requested 2024-12-09T06:19:15,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155000 2024-12-09T06:19:15,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725154993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155000 2024-12-09T06:19:15,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155000) roll requested 2024-12-09T06:19:15,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155006 2024-12-09T06:19:15,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155006 2024-12-09T06:19:15,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155006) roll requested 2024-12-09T06:19:15,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155013 2024-12-09T06:19:15,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155013 2024-12-09T06:19:15,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155013) roll requested 2024-12-09T06:19:15,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155018 2024-12-09T06:19:15,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155018 2024-12-09T06:19:15,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155018) roll requested 2024-12-09T06:19:15,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155024 2024-12-09T06:19:15,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155024 2024-12-09T06:19:15,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155024) roll requested 2024-12-09T06:19:15,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155031 2024-12-09T06:19:15,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155031 2024-12-09T06:19:15,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155031) roll requested 2024-12-09T06:19:15,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155036 2024-12-09T06:19:15,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155036 2024-12-09T06:19:15,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155036) roll requested 2024-12-09T06:19:15,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155042 2024-12-09T06:19:15,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155042 2024-12-09T06:19:15,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155042) roll requested 2024-12-09T06:19:15,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155048 2024-12-09T06:19:15,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155048 2024-12-09T06:19:15,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155048) roll requested 2024-12-09T06:19:15,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155053 2024-12-09T06:19:15,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155053 2024-12-09T06:19:15,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155053) roll requested 2024-12-09T06:19:15,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155057 2024-12-09T06:19:15,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155057 2024-12-09T06:19:15,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155057) roll requested 2024-12-09T06:19:15,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155063 2024-12-09T06:19:15,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155063 2024-12-09T06:19:15,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155063) roll requested 2024-12-09T06:19:15,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155069 2024-12-09T06:19:15,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155069 2024-12-09T06:19:15,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155069) roll requested 2024-12-09T06:19:15,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155075 2024-12-09T06:19:15,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155075 2024-12-09T06:19:15,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155075) roll requested 2024-12-09T06:19:15,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155082 2024-12-09T06:19:15,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155082 2024-12-09T06:19:15,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155082) roll requested 2024-12-09T06:19:15,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155088 2024-12-09T06:19:15,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155088 2024-12-09T06:19:15,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155088) roll requested 2024-12-09T06:19:15,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155094 2024-12-09T06:19:15,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155094 2024-12-09T06:19:15,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155094) roll requested 2024-12-09T06:19:15,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155099 2024-12-09T06:19:15,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155099 2024-12-09T06:19:15,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155099) roll requested 2024-12-09T06:19:15,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155105 2024-12-09T06:19:15,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155105 2024-12-09T06:19:15,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155105) roll requested 2024-12-09T06:19:15,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155109 2024-12-09T06:19:15,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155109 2024-12-09T06:19:15,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155109) roll requested 2024-12-09T06:19:15,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155115 2024-12-09T06:19:15,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155115 2024-12-09T06:19:15,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155115) roll requested 2024-12-09T06:19:15,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155122 2024-12-09T06:19:15,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155122 2024-12-09T06:19:15,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155122) roll requested 2024-12-09T06:19:15,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155128 2024-12-09T06:19:15,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155128 2024-12-09T06:19:15,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155128) roll requested 2024-12-09T06:19:15,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155135 2024-12-09T06:19:15,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155135 2024-12-09T06:19:15,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155135) roll requested 2024-12-09T06:19:15,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155142 2024-12-09T06:19:15,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155142 2024-12-09T06:19:15,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155142) roll requested 2024-12-09T06:19:15,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155150 2024-12-09T06:19:15,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155150 2024-12-09T06:19:15,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155150) roll requested 2024-12-09T06:19:15,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155156 2024-12-09T06:19:15,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155156 2024-12-09T06:19:15,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155156) roll requested 2024-12-09T06:19:15,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155163 2024-12-09T06:19:15,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155163 2024-12-09T06:19:15,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155163) roll requested 2024-12-09T06:19:15,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155171 2024-12-09T06:19:15,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155171 2024-12-09T06:19:15,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155171) roll requested 2024-12-09T06:19:15,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155180 2024-12-09T06:19:15,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155180 2024-12-09T06:19:15,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155180) roll requested 2024-12-09T06:19:15,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155190 2024-12-09T06:19:15,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155190 2024-12-09T06:19:15,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155190) roll requested 2024-12-09T06:19:15,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155195 2024-12-09T06:19:15,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155195 2024-12-09T06:19:15,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155195) roll requested 2024-12-09T06:19:15,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155200 2024-12-09T06:19:15,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155200 2024-12-09T06:19:15,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155200) roll requested 2024-12-09T06:19:15,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155206 2024-12-09T06:19:15,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155206 2024-12-09T06:19:15,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155206) roll requested 2024-12-09T06:19:15,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155215 2024-12-09T06:19:15,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155215 2024-12-09T06:19:15,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155215) roll requested 2024-12-09T06:19:15,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155220 2024-12-09T06:19:15,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155220 2024-12-09T06:19:15,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155220) roll requested 2024-12-09T06:19:15,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155225 2024-12-09T06:19:15,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155225 2024-12-09T06:19:15,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155225) roll requested 2024-12-09T06:19:15,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155231 2024-12-09T06:19:15,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155231 2024-12-09T06:19:15,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155231) roll requested 2024-12-09T06:19:15,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155237 2024-12-09T06:19:15,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155237 2024-12-09T06:19:15,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155237) roll requested 2024-12-09T06:19:15,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155244 2024-12-09T06:19:15,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155244 2024-12-09T06:19:15,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155244) roll requested 2024-12-09T06:19:15,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155251 2024-12-09T06:19:15,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155251 2024-12-09T06:19:15,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155251) roll requested 2024-12-09T06:19:15,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155260 2024-12-09T06:19:15,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155260 2024-12-09T06:19:15,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155260) roll requested 2024-12-09T06:19:15,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155268 2024-12-09T06:19:15,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155268 2024-12-09T06:19:15,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155268) roll requested 2024-12-09T06:19:15,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155275 2024-12-09T06:19:15,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155275 2024-12-09T06:19:15,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155275) roll requested 2024-12-09T06:19:15,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155281 2024-12-09T06:19:15,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155281 2024-12-09T06:19:15,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155281) roll requested 2024-12-09T06:19:15,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155288 2024-12-09T06:19:15,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155288 2024-12-09T06:19:15,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155288) roll requested 2024-12-09T06:19:15,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155297 2024-12-09T06:19:15,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155297 2024-12-09T06:19:15,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155297) roll requested 2024-12-09T06:19:15,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155306 2024-12-09T06:19:15,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155306 2024-12-09T06:19:15,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155306) roll requested 2024-12-09T06:19:15,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155315 2024-12-09T06:19:15,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155315 2024-12-09T06:19:15,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155315) roll requested 2024-12-09T06:19:15,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155320 2024-12-09T06:19:15,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155320 2024-12-09T06:19:15,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155320) roll requested 2024-12-09T06:19:15,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155324 2024-12-09T06:19:15,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155324 2024-12-09T06:19:15,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155324) roll requested 2024-12-09T06:19:15,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155330 2024-12-09T06:19:15,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155330 2024-12-09T06:19:15,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155330) roll requested 2024-12-09T06:19:15,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155338 2024-12-09T06:19:15,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155338 2024-12-09T06:19:15,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155338) roll requested 2024-12-09T06:19:15,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155347 2024-12-09T06:19:15,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155347 2024-12-09T06:19:15,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155347) roll requested 2024-12-09T06:19:15,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155353 2024-12-09T06:19:15,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155353 2024-12-09T06:19:15,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155353) roll requested 2024-12-09T06:19:15,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155359 2024-12-09T06:19:15,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155359 2024-12-09T06:19:15,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155359) roll requested 2024-12-09T06:19:15,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155366 2024-12-09T06:19:15,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155366 2024-12-09T06:19:15,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155366) roll requested 2024-12-09T06:19:15,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155374 2024-12-09T06:19:15,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155374 2024-12-09T06:19:15,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155374) roll requested 2024-12-09T06:19:15,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155384 2024-12-09T06:19:15,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155384 2024-12-09T06:19:15,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155384) roll requested 2024-12-09T06:19:15,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155389 2024-12-09T06:19:15,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155389 2024-12-09T06:19:15,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155389) roll requested 2024-12-09T06:19:15,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155394 2024-12-09T06:19:15,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155394 2024-12-09T06:19:15,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155394) roll requested 2024-12-09T06:19:15,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155399 2024-12-09T06:19:15,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155399 2024-12-09T06:19:15,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155399) roll requested 2024-12-09T06:19:15,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155406 2024-12-09T06:19:15,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155406 2024-12-09T06:19:15,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155406) roll requested 2024-12-09T06:19:15,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155414 2024-12-09T06:19:15,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155414 2024-12-09T06:19:15,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155414) roll requested 2024-12-09T06:19:15,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155420 2024-12-09T06:19:15,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155420 2024-12-09T06:19:15,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155420) roll requested 2024-12-09T06:19:15,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155426 2024-12-09T06:19:15,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155426 2024-12-09T06:19:15,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155426) roll requested 2024-12-09T06:19:15,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155434 2024-12-09T06:19:15,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155434 2024-12-09T06:19:15,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155434) roll requested 2024-12-09T06:19:15,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155441 2024-12-09T06:19:15,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155441 2024-12-09T06:19:15,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155441) roll requested 2024-12-09T06:19:15,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155449 2024-12-09T06:19:15,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155449 2024-12-09T06:19:15,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155449) roll requested 2024-12-09T06:19:15,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155458 2024-12-09T06:19:15,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155458 2024-12-09T06:19:15,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155458) roll requested 2024-12-09T06:19:15,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155468 2024-12-09T06:19:15,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155468 2024-12-09T06:19:15,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155468) roll requested 2024-12-09T06:19:15,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155476 2024-12-09T06:19:15,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155476 2024-12-09T06:19:15,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155476) roll requested 2024-12-09T06:19:15,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155485 2024-12-09T06:19:15,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155485 2024-12-09T06:19:15,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155485) roll requested 2024-12-09T06:19:15,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155494 2024-12-09T06:19:15,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155494 2024-12-09T06:19:15,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155494) roll requested 2024-12-09T06:19:15,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155503 2024-12-09T06:19:15,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155503 2024-12-09T06:19:15,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155503) roll requested 2024-12-09T06:19:15,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155510 2024-12-09T06:19:15,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155510 2024-12-09T06:19:15,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155510) roll requested 2024-12-09T06:19:15,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155521 2024-12-09T06:19:15,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155521 2024-12-09T06:19:15,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155521) roll requested 2024-12-09T06:19:15,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155530 2024-12-09T06:19:15,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155530 2024-12-09T06:19:15,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155530) roll requested 2024-12-09T06:19:15,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155535 2024-12-09T06:19:15,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155535 2024-12-09T06:19:15,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155535) roll requested 2024-12-09T06:19:15,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155540 2024-12-09T06:19:15,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155540 2024-12-09T06:19:15,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155540) roll requested 2024-12-09T06:19:15,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155546 2024-12-09T06:19:15,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155546 2024-12-09T06:19:15,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155546) roll requested 2024-12-09T06:19:15,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155552 2024-12-09T06:19:15,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155552 2024-12-09T06:19:15,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155552) roll requested 2024-12-09T06:19:15,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155559 2024-12-09T06:19:15,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155559 2024-12-09T06:19:15,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155559) roll requested 2024-12-09T06:19:15,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155566 2024-12-09T06:19:15,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155566 2024-12-09T06:19:15,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155566) roll requested 2024-12-09T06:19:15,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155575 2024-12-09T06:19:15,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155575 2024-12-09T06:19:15,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155575) roll requested 2024-12-09T06:19:15,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155584 2024-12-09T06:19:15,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155584 2024-12-09T06:19:15,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155584) roll requested 2024-12-09T06:19:15,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155594 2024-12-09T06:19:15,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155594 2024-12-09T06:19:15,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155594) roll requested 2024-12-09T06:19:15,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155603 2024-12-09T06:19:15,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155603 2024-12-09T06:19:15,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155603) roll requested 2024-12-09T06:19:15,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155607 2024-12-09T06:19:15,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155607 2024-12-09T06:19:15,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155607) roll requested 2024-12-09T06:19:15,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155614 2024-12-09T06:19:15,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155614 2024-12-09T06:19:15,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155614) roll requested 2024-12-09T06:19:15,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155623 2024-12-09T06:19:15,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155623 2024-12-09T06:19:15,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155623) roll requested 2024-12-09T06:19:15,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155637 2024-12-09T06:19:15,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155637 2024-12-09T06:19:15,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155637) roll requested 2024-12-09T06:19:15,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155649 2024-12-09T06:19:15,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155649 2024-12-09T06:19:15,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155649) roll requested 2024-12-09T06:19:15,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155655 2024-12-09T06:19:15,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155655 2024-12-09T06:19:15,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155655) roll requested 2024-12-09T06:19:15,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155660 2024-12-09T06:19:15,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155660 2024-12-09T06:19:15,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155660) roll requested 2024-12-09T06:19:15,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155666 2024-12-09T06:19:15,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155666 2024-12-09T06:19:15,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155666) roll requested 2024-12-09T06:19:15,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155670 2024-12-09T06:19:15,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155670 2024-12-09T06:19:15,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155670) roll requested 2024-12-09T06:19:15,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155676 2024-12-09T06:19:15,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155676 2024-12-09T06:19:15,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155676) roll requested 2024-12-09T06:19:15,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155681 2024-12-09T06:19:15,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155681 2024-12-09T06:19:15,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155681) roll requested 2024-12-09T06:19:15,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155689 2024-12-09T06:19:15,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155689 2024-12-09T06:19:15,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155689) roll requested 2024-12-09T06:19:15,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155697 2024-12-09T06:19:15,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155697 2024-12-09T06:19:15,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155697) roll requested 2024-12-09T06:19:15,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155705 2024-12-09T06:19:15,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155705 2024-12-09T06:19:15,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155705) roll requested 2024-12-09T06:19:15,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155715 2024-12-09T06:19:15,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155715 2024-12-09T06:19:15,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155715) roll requested 2024-12-09T06:19:15,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155728 2024-12-09T06:19:15,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155728 2024-12-09T06:19:15,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155728) roll requested 2024-12-09T06:19:15,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155734 2024-12-09T06:19:15,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155734 2024-12-09T06:19:15,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155734) roll requested 2024-12-09T06:19:15,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155739 2024-12-09T06:19:15,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155739 2024-12-09T06:19:15,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155739) roll requested 2024-12-09T06:19:15,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155745 2024-12-09T06:19:15,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155745 2024-12-09T06:19:15,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155745) roll requested 2024-12-09T06:19:15,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155750 2024-12-09T06:19:15,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155750 2024-12-09T06:19:15,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155750) roll requested 2024-12-09T06:19:15,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155758 2024-12-09T06:19:15,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155758 2024-12-09T06:19:15,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155758) roll requested 2024-12-09T06:19:15,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155765 2024-12-09T06:19:15,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155765 2024-12-09T06:19:15,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155765) roll requested 2024-12-09T06:19:15,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155771 2024-12-09T06:19:15,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155771 2024-12-09T06:19:15,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155771) roll requested 2024-12-09T06:19:15,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155777 2024-12-09T06:19:15,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155777 2024-12-09T06:19:15,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155777) roll requested 2024-12-09T06:19:15,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155783 2024-12-09T06:19:15,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155783 2024-12-09T06:19:15,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155783) roll requested 2024-12-09T06:19:15,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155789 2024-12-09T06:19:15,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155789 2024-12-09T06:19:15,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155789) roll requested 2024-12-09T06:19:15,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155796 2024-12-09T06:19:15,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155796 2024-12-09T06:19:15,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155796) roll requested 2024-12-09T06:19:15,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155804 2024-12-09T06:19:15,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155804 2024-12-09T06:19:15,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155804) roll requested 2024-12-09T06:19:15,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155813 2024-12-09T06:19:15,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155813 2024-12-09T06:19:15,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155813) roll requested 2024-12-09T06:19:15,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155823 2024-12-09T06:19:15,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155823 2024-12-09T06:19:15,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155823) roll requested 2024-12-09T06:19:15,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155832 2024-12-09T06:19:15,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155832 2024-12-09T06:19:15,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155832) roll requested 2024-12-09T06:19:15,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155840 2024-12-09T06:19:15,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155840 2024-12-09T06:19:15,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155840) roll requested 2024-12-09T06:19:15,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155847 2024-12-09T06:19:15,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155847 2024-12-09T06:19:15,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155847) roll requested 2024-12-09T06:19:15,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155856 2024-12-09T06:19:15,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155856 2024-12-09T06:19:15,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155856) roll requested 2024-12-09T06:19:15,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155862 2024-12-09T06:19:15,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155862 2024-12-09T06:19:15,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155862) roll requested 2024-12-09T06:19:15,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155867 2024-12-09T06:19:15,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155867 2024-12-09T06:19:15,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155867) roll requested 2024-12-09T06:19:15,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155872 2024-12-09T06:19:15,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155872 2024-12-09T06:19:15,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155872) roll requested 2024-12-09T06:19:15,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155876 2024-12-09T06:19:15,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155876 2024-12-09T06:19:15,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155876) roll requested 2024-12-09T06:19:15,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155881 2024-12-09T06:19:15,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155881 2024-12-09T06:19:15,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155881) roll requested 2024-12-09T06:19:15,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155888 2024-12-09T06:19:15,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155888 2024-12-09T06:19:15,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155888) roll requested 2024-12-09T06:19:15,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155895 2024-12-09T06:19:15,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155895 2024-12-09T06:19:15,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155895) roll requested 2024-12-09T06:19:15,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155903 2024-12-09T06:19:15,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155903 2024-12-09T06:19:15,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155903) roll requested 2024-12-09T06:19:15,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155910 2024-12-09T06:19:15,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155910 2024-12-09T06:19:15,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155910) roll requested 2024-12-09T06:19:15,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155921 2024-12-09T06:19:15,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155921 2024-12-09T06:19:15,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155921) roll requested 2024-12-09T06:19:15,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155929 2024-12-09T06:19:15,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155929 2024-12-09T06:19:15,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155929) roll requested 2024-12-09T06:19:15,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155939 2024-12-09T06:19:15,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155939 2024-12-09T06:19:15,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155939) roll requested 2024-12-09T06:19:15,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155945 2024-12-09T06:19:15,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155945 2024-12-09T06:19:15,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155945) roll requested 2024-12-09T06:19:15,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155952 2024-12-09T06:19:15,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155952 2024-12-09T06:19:15,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155952) roll requested 2024-12-09T06:19:15,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155958 2024-12-09T06:19:15,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155958 2024-12-09T06:19:15,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155958) roll requested 2024-12-09T06:19:15,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155966 2024-12-09T06:19:15,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155966 2024-12-09T06:19:15,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155966) roll requested 2024-12-09T06:19:15,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155974 2024-12-09T06:19:15,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155974 2024-12-09T06:19:15,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155974) roll requested 2024-12-09T06:19:15,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155982 2024-12-09T06:19:15,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155982 2024-12-09T06:19:15,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155982) roll requested 2024-12-09T06:19:15,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155987 2024-12-09T06:19:15,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155987 2024-12-09T06:19:15,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155987) roll requested 2024-12-09T06:19:15,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155992 2024-12-09T06:19:15,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:15,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155992 2024-12-09T06:19:15,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:15,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:15,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155992) roll requested 2024-12-09T06:19:15,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725155999 2024-12-09T06:19:16,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155999 2024-12-09T06:19:16,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725155999) roll requested 2024-12-09T06:19:16,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156006 2024-12-09T06:19:16,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725155999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156006 2024-12-09T06:19:16,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156006) roll requested 2024-12-09T06:19:16,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156014 2024-12-09T06:19:16,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156014 2024-12-09T06:19:16,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156014) roll requested 2024-12-09T06:19:16,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156023 2024-12-09T06:19:16,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156023 2024-12-09T06:19:16,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156023) roll requested 2024-12-09T06:19:16,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156029 2024-12-09T06:19:16,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156029 2024-12-09T06:19:16,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156029) roll requested 2024-12-09T06:19:16,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156036 2024-12-09T06:19:16,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156036 2024-12-09T06:19:16,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156036) roll requested 2024-12-09T06:19:16,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156041 2024-12-09T06:19:16,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156041 2024-12-09T06:19:16,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156041) roll requested 2024-12-09T06:19:16,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156047 2024-12-09T06:19:16,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156047 2024-12-09T06:19:16,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156047) roll requested 2024-12-09T06:19:16,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156054 2024-12-09T06:19:16,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156054 2024-12-09T06:19:16,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156054) roll requested 2024-12-09T06:19:16,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156061 2024-12-09T06:19:16,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156061 2024-12-09T06:19:16,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156061) roll requested 2024-12-09T06:19:16,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156069 2024-12-09T06:19:16,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156069 2024-12-09T06:19:16,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156069) roll requested 2024-12-09T06:19:16,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156077 2024-12-09T06:19:16,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156077 2024-12-09T06:19:16,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156077) roll requested 2024-12-09T06:19:16,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156084 2024-12-09T06:19:16,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156084 2024-12-09T06:19:16,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156084) roll requested 2024-12-09T06:19:16,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156094 2024-12-09T06:19:16,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156094 2024-12-09T06:19:16,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156094) roll requested 2024-12-09T06:19:16,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156100 2024-12-09T06:19:16,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156100 2024-12-09T06:19:16,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156100) roll requested 2024-12-09T06:19:16,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156106 2024-12-09T06:19:16,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156106 2024-12-09T06:19:16,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156106) roll requested 2024-12-09T06:19:16,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156112 2024-12-09T06:19:16,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156112 2024-12-09T06:19:16,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156112) roll requested 2024-12-09T06:19:16,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156119 2024-12-09T06:19:16,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156119 2024-12-09T06:19:16,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156119) roll requested 2024-12-09T06:19:16,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156126 2024-12-09T06:19:16,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156126 2024-12-09T06:19:16,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156126) roll requested 2024-12-09T06:19:16,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156132 2024-12-09T06:19:16,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156132 2024-12-09T06:19:16,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156132) roll requested 2024-12-09T06:19:16,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156151 2024-12-09T06:19:16,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156151 2024-12-09T06:19:16,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156151) roll requested 2024-12-09T06:19:16,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156158 2024-12-09T06:19:16,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156158 2024-12-09T06:19:16,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156158) roll requested 2024-12-09T06:19:16,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156163 2024-12-09T06:19:16,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156163 2024-12-09T06:19:16,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156163) roll requested 2024-12-09T06:19:16,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156168 2024-12-09T06:19:16,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156168 2024-12-09T06:19:16,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156168) roll requested 2024-12-09T06:19:16,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156172 2024-12-09T06:19:16,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156172 2024-12-09T06:19:16,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156172) roll requested 2024-12-09T06:19:16,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156178 2024-12-09T06:19:16,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156178 2024-12-09T06:19:16,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156178) roll requested 2024-12-09T06:19:16,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156184 2024-12-09T06:19:16,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156184 2024-12-09T06:19:16,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156184) roll requested 2024-12-09T06:19:16,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156190 2024-12-09T06:19:16,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156190 2024-12-09T06:19:16,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156190) roll requested 2024-12-09T06:19:16,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156197 2024-12-09T06:19:16,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156197 2024-12-09T06:19:16,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156197) roll requested 2024-12-09T06:19:16,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156205 2024-12-09T06:19:16,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156205 2024-12-09T06:19:16,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156205) roll requested 2024-12-09T06:19:16,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156211 2024-12-09T06:19:16,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156211 2024-12-09T06:19:16,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156211) roll requested 2024-12-09T06:19:16,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156216 2024-12-09T06:19:16,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156216 2024-12-09T06:19:16,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156216) roll requested 2024-12-09T06:19:16,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156223 2024-12-09T06:19:16,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156223 2024-12-09T06:19:16,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156223) roll requested 2024-12-09T06:19:16,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156231 2024-12-09T06:19:16,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156231 2024-12-09T06:19:16,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156231) roll requested 2024-12-09T06:19:16,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156239 2024-12-09T06:19:16,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156239 2024-12-09T06:19:16,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156239) roll requested 2024-12-09T06:19:16,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156248 2024-12-09T06:19:16,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156248 2024-12-09T06:19:16,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156248) roll requested 2024-12-09T06:19:16,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156256 2024-12-09T06:19:16,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156256 2024-12-09T06:19:16,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156256) roll requested 2024-12-09T06:19:16,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156263 2024-12-09T06:19:16,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156263 2024-12-09T06:19:16,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156263) roll requested 2024-12-09T06:19:16,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156272 2024-12-09T06:19:16,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156272 2024-12-09T06:19:16,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156272) roll requested 2024-12-09T06:19:16,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156279 2024-12-09T06:19:16,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156279 2024-12-09T06:19:16,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156279) roll requested 2024-12-09T06:19:16,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156286 2024-12-09T06:19:16,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156286 2024-12-09T06:19:16,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156286) roll requested 2024-12-09T06:19:16,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156293 2024-12-09T06:19:16,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156293 2024-12-09T06:19:16,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156293) roll requested 2024-12-09T06:19:16,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156302 2024-12-09T06:19:16,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156302 2024-12-09T06:19:16,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156302) roll requested 2024-12-09T06:19:16,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156310 2024-12-09T06:19:16,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156310 2024-12-09T06:19:16,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156310) roll requested 2024-12-09T06:19:16,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156317 2024-12-09T06:19:16,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156317 2024-12-09T06:19:16,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156317) roll requested 2024-12-09T06:19:16,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156324 2024-12-09T06:19:16,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156324 2024-12-09T06:19:16,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156324) roll requested 2024-12-09T06:19:16,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156333 2024-12-09T06:19:16,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156333 2024-12-09T06:19:16,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156333) roll requested 2024-12-09T06:19:16,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156342 2024-12-09T06:19:16,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156342 2024-12-09T06:19:16,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156342) roll requested 2024-12-09T06:19:16,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156348 2024-12-09T06:19:16,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156348 2024-12-09T06:19:16,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156348) roll requested 2024-12-09T06:19:16,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156355 2024-12-09T06:19:16,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156355 2024-12-09T06:19:16,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156355) roll requested 2024-12-09T06:19:16,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156363 2024-12-09T06:19:16,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156363 2024-12-09T06:19:16,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156363) roll requested 2024-12-09T06:19:16,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156374 2024-12-09T06:19:16,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156374 2024-12-09T06:19:16,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156374) roll requested 2024-12-09T06:19:16,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156379 2024-12-09T06:19:16,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156379 2024-12-09T06:19:16,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156379) roll requested 2024-12-09T06:19:16,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156383 2024-12-09T06:19:16,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156383 2024-12-09T06:19:16,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156383) roll requested 2024-12-09T06:19:16,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156390 2024-12-09T06:19:16,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156390 2024-12-09T06:19:16,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156390) roll requested 2024-12-09T06:19:16,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156398 2024-12-09T06:19:16,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156398 2024-12-09T06:19:16,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156398) roll requested 2024-12-09T06:19:16,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156405 2024-12-09T06:19:16,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156405 2024-12-09T06:19:16,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156405) roll requested 2024-12-09T06:19:16,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156412 2024-12-09T06:19:16,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156412 2024-12-09T06:19:16,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156412) roll requested 2024-12-09T06:19:16,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156419 2024-12-09T06:19:16,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156419 2024-12-09T06:19:16,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156419) roll requested 2024-12-09T06:19:16,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156425 2024-12-09T06:19:16,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156425 2024-12-09T06:19:16,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156425) roll requested 2024-12-09T06:19:16,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156432 2024-12-09T06:19:16,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156432 2024-12-09T06:19:16,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156432) roll requested 2024-12-09T06:19:16,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156440 2024-12-09T06:19:16,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156440 2024-12-09T06:19:16,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156440) roll requested 2024-12-09T06:19:16,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156449 2024-12-09T06:19:16,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156449 2024-12-09T06:19:16,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156449) roll requested 2024-12-09T06:19:16,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156455 2024-12-09T06:19:16,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156455 2024-12-09T06:19:16,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156455) roll requested 2024-12-09T06:19:16,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156461 2024-12-09T06:19:16,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156461 2024-12-09T06:19:16,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156461) roll requested 2024-12-09T06:19:16,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156469 2024-12-09T06:19:16,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156469 2024-12-09T06:19:16,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156469) roll requested 2024-12-09T06:19:16,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156477 2024-12-09T06:19:16,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156477 2024-12-09T06:19:16,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156477) roll requested 2024-12-09T06:19:16,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156485 2024-12-09T06:19:16,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156485 2024-12-09T06:19:16,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156485) roll requested 2024-12-09T06:19:16,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156495 2024-12-09T06:19:16,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156495 2024-12-09T06:19:16,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156495) roll requested 2024-12-09T06:19:16,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156504 2024-12-09T06:19:16,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156504 2024-12-09T06:19:16,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156504) roll requested 2024-12-09T06:19:16,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156512 2024-12-09T06:19:16,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156512 2024-12-09T06:19:16,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156512) roll requested 2024-12-09T06:19:16,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156521 2024-12-09T06:19:16,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156521 2024-12-09T06:19:16,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156521) roll requested 2024-12-09T06:19:16,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156531 2024-12-09T06:19:16,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156531 2024-12-09T06:19:16,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156531) roll requested 2024-12-09T06:19:16,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156541 2024-12-09T06:19:16,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156541 2024-12-09T06:19:16,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156541) roll requested 2024-12-09T06:19:16,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156551 2024-12-09T06:19:16,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156551 2024-12-09T06:19:16,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156551) roll requested 2024-12-09T06:19:16,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156561 2024-12-09T06:19:16,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156561 2024-12-09T06:19:16,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156561) roll requested 2024-12-09T06:19:16,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156569 2024-12-09T06:19:16,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156569 2024-12-09T06:19:16,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156569) roll requested 2024-12-09T06:19:16,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156577 2024-12-09T06:19:16,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156577 2024-12-09T06:19:16,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156577) roll requested 2024-12-09T06:19:16,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156585 2024-12-09T06:19:16,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156585 2024-12-09T06:19:16,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156585) roll requested 2024-12-09T06:19:16,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156591 2024-12-09T06:19:16,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156591 2024-12-09T06:19:16,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156591) roll requested 2024-12-09T06:19:16,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156598 2024-12-09T06:19:16,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156598 2024-12-09T06:19:16,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156598) roll requested 2024-12-09T06:19:16,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156607 2024-12-09T06:19:16,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156607 2024-12-09T06:19:16,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156607) roll requested 2024-12-09T06:19:16,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156616 2024-12-09T06:19:16,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156616 2024-12-09T06:19:16,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156616) roll requested 2024-12-09T06:19:16,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156622 2024-12-09T06:19:16,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156622 2024-12-09T06:19:16,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156622) roll requested 2024-12-09T06:19:16,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156629 2024-12-09T06:19:16,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156629 2024-12-09T06:19:16,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156629) roll requested 2024-12-09T06:19:16,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156636 2024-12-09T06:19:16,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156636 2024-12-09T06:19:16,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156636) roll requested 2024-12-09T06:19:16,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156643 2024-12-09T06:19:16,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156643 2024-12-09T06:19:16,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156643) roll requested 2024-12-09T06:19:16,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156652 2024-12-09T06:19:16,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156652 2024-12-09T06:19:16,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156652) roll requested 2024-12-09T06:19:16,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156665 2024-12-09T06:19:16,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156665 2024-12-09T06:19:16,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156665) roll requested 2024-12-09T06:19:16,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156676 2024-12-09T06:19:16,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156676 2024-12-09T06:19:16,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156676) roll requested 2024-12-09T06:19:16,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156681 2024-12-09T06:19:16,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156681 2024-12-09T06:19:16,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156681) roll requested 2024-12-09T06:19:16,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156689 2024-12-09T06:19:16,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156689 2024-12-09T06:19:16,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156689) roll requested 2024-12-09T06:19:16,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156694 2024-12-09T06:19:16,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156694 2024-12-09T06:19:16,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156694) roll requested 2024-12-09T06:19:16,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156698 2024-12-09T06:19:16,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156698 2024-12-09T06:19:16,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156698) roll requested 2024-12-09T06:19:16,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156703 2024-12-09T06:19:16,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156703 2024-12-09T06:19:16,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156703) roll requested 2024-12-09T06:19:16,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156708 2024-12-09T06:19:16,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156708 2024-12-09T06:19:16,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156708) roll requested 2024-12-09T06:19:16,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156715 2024-12-09T06:19:16,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156715 2024-12-09T06:19:16,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156715) roll requested 2024-12-09T06:19:16,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156722 2024-12-09T06:19:16,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156722 2024-12-09T06:19:16,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156722) roll requested 2024-12-09T06:19:16,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156729 2024-12-09T06:19:16,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156729 2024-12-09T06:19:16,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156729) roll requested 2024-12-09T06:19:16,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156738 2024-12-09T06:19:16,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156738 2024-12-09T06:19:16,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156738) roll requested 2024-12-09T06:19:16,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156745 2024-12-09T06:19:16,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156745 2024-12-09T06:19:16,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156745) roll requested 2024-12-09T06:19:16,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156753 2024-12-09T06:19:16,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156753 2024-12-09T06:19:16,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156753) roll requested 2024-12-09T06:19:16,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156762 2024-12-09T06:19:16,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156762 2024-12-09T06:19:16,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156762) roll requested 2024-12-09T06:19:16,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156771 2024-12-09T06:19:16,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156771 2024-12-09T06:19:16,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156771) roll requested 2024-12-09T06:19:16,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156778 2024-12-09T06:19:16,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156778 2024-12-09T06:19:16,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156778) roll requested 2024-12-09T06:19:16,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156787 2024-12-09T06:19:16,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156787 2024-12-09T06:19:16,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156787) roll requested 2024-12-09T06:19:16,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156795 2024-12-09T06:19:16,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156795 2024-12-09T06:19:16,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156795) roll requested 2024-12-09T06:19:16,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156804 2024-12-09T06:19:16,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156804 2024-12-09T06:19:16,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156804) roll requested 2024-12-09T06:19:16,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156810 2024-12-09T06:19:16,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156810 2024-12-09T06:19:16,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156810) roll requested 2024-12-09T06:19:16,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156816 2024-12-09T06:19:16,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156816 2024-12-09T06:19:16,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156816) roll requested 2024-12-09T06:19:16,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156823 2024-12-09T06:19:16,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156823 2024-12-09T06:19:16,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156823) roll requested 2024-12-09T06:19:16,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156830 2024-12-09T06:19:16,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156830 2024-12-09T06:19:16,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156830) roll requested 2024-12-09T06:19:16,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156837 2024-12-09T06:19:16,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156837 2024-12-09T06:19:16,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156837) roll requested 2024-12-09T06:19:16,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156843 2024-12-09T06:19:16,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156843 2024-12-09T06:19:16,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156843) roll requested 2024-12-09T06:19:16,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156850 2024-12-09T06:19:16,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156850 2024-12-09T06:19:16,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156850) roll requested 2024-12-09T06:19:16,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156858 2024-12-09T06:19:16,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156858 2024-12-09T06:19:16,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156858) roll requested 2024-12-09T06:19:16,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156867 2024-12-09T06:19:16,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156867 2024-12-09T06:19:16,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156867) roll requested 2024-12-09T06:19:16,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156876 2024-12-09T06:19:16,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156876 2024-12-09T06:19:16,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156876) roll requested 2024-12-09T06:19:16,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156885 2024-12-09T06:19:16,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156885 2024-12-09T06:19:16,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156885) roll requested 2024-12-09T06:19:16,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156892 2024-12-09T06:19:16,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156892 2024-12-09T06:19:16,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156892) roll requested 2024-12-09T06:19:16,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156898 2024-12-09T06:19:16,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156898 2024-12-09T06:19:16,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156898) roll requested 2024-12-09T06:19:16,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156907 2024-12-09T06:19:16,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156907 2024-12-09T06:19:16,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156907) roll requested 2024-12-09T06:19:16,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156916 2024-12-09T06:19:16,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156916 2024-12-09T06:19:16,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156916) roll requested 2024-12-09T06:19:16,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156922 2024-12-09T06:19:16,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156922 2024-12-09T06:19:16,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156922) roll requested 2024-12-09T06:19:16,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156929 2024-12-09T06:19:16,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156929 2024-12-09T06:19:16,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156929) roll requested 2024-12-09T06:19:16,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156937 2024-12-09T06:19:16,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156937 2024-12-09T06:19:16,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156937) roll requested 2024-12-09T06:19:16,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156943 2024-12-09T06:19:16,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156943 2024-12-09T06:19:16,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156943) roll requested 2024-12-09T06:19:16,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156948 2024-12-09T06:19:16,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156948 2024-12-09T06:19:16,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156948) roll requested 2024-12-09T06:19:16,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156953 2024-12-09T06:19:16,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156953 2024-12-09T06:19:16,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156953) roll requested 2024-12-09T06:19:16,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156959 2024-12-09T06:19:16,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156959 2024-12-09T06:19:16,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156959) roll requested 2024-12-09T06:19:16,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156966 2024-12-09T06:19:16,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156966 2024-12-09T06:19:16,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156966) roll requested 2024-12-09T06:19:16,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156973 2024-12-09T06:19:16,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156973 2024-12-09T06:19:16,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156973) roll requested 2024-12-09T06:19:16,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156981 2024-12-09T06:19:16,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156981 2024-12-09T06:19:16,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156981) roll requested 2024-12-09T06:19:16,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156990 2024-12-09T06:19:16,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:16,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156990 2024-12-09T06:19:16,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:16,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:16,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156990) roll requested 2024-12-09T06:19:16,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725156995 2024-12-09T06:19:16,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156995 2024-12-09T06:19:17,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725156995) roll requested 2024-12-09T06:19:17,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157001 2024-12-09T06:19:17,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725156995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157001 2024-12-09T06:19:17,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157001) roll requested 2024-12-09T06:19:17,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157008 2024-12-09T06:19:17,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157008 2024-12-09T06:19:17,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157008) roll requested 2024-12-09T06:19:17,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157016 2024-12-09T06:19:17,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157016 2024-12-09T06:19:17,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157016) roll requested 2024-12-09T06:19:17,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157026 2024-12-09T06:19:17,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157026 2024-12-09T06:19:17,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157026) roll requested 2024-12-09T06:19:17,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157033 2024-12-09T06:19:17,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157033 2024-12-09T06:19:17,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157033) roll requested 2024-12-09T06:19:17,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157038 2024-12-09T06:19:17,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157038 2024-12-09T06:19:17,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157038) roll requested 2024-12-09T06:19:17,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157045 2024-12-09T06:19:17,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157045 2024-12-09T06:19:17,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157045) roll requested 2024-12-09T06:19:17,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157051 2024-12-09T06:19:17,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157051 2024-12-09T06:19:17,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157051) roll requested 2024-12-09T06:19:17,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157057 2024-12-09T06:19:17,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157057 2024-12-09T06:19:17,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157057) roll requested 2024-12-09T06:19:17,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157064 2024-12-09T06:19:17,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157064 2024-12-09T06:19:17,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157064) roll requested 2024-12-09T06:19:17,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157072 2024-12-09T06:19:17,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157072 2024-12-09T06:19:17,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157072) roll requested 2024-12-09T06:19:17,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157080 2024-12-09T06:19:17,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157080 2024-12-09T06:19:17,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157080) roll requested 2024-12-09T06:19:17,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157090 2024-12-09T06:19:17,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157090 2024-12-09T06:19:17,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157090) roll requested 2024-12-09T06:19:17,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157095 2024-12-09T06:19:17,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157095 2024-12-09T06:19:17,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157095) roll requested 2024-12-09T06:19:17,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157102 2024-12-09T06:19:17,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157102 2024-12-09T06:19:17,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157102) roll requested 2024-12-09T06:19:17,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157109 2024-12-09T06:19:17,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157109 2024-12-09T06:19:17,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157109) roll requested 2024-12-09T06:19:17,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157120 2024-12-09T06:19:17,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157120 2024-12-09T06:19:17,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157120) roll requested 2024-12-09T06:19:17,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157128 2024-12-09T06:19:17,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157128 2024-12-09T06:19:17,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157128) roll requested 2024-12-09T06:19:17,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157136 2024-12-09T06:19:17,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157136 2024-12-09T06:19:17,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157136) roll requested 2024-12-09T06:19:17,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157144 2024-12-09T06:19:17,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157144 2024-12-09T06:19:17,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157144) roll requested 2024-12-09T06:19:17,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157153 2024-12-09T06:19:17,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157153 2024-12-09T06:19:17,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157153) roll requested 2024-12-09T06:19:17,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157158 2024-12-09T06:19:17,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157158 2024-12-09T06:19:17,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157158) roll requested 2024-12-09T06:19:17,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157163 2024-12-09T06:19:17,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157163 2024-12-09T06:19:17,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157163) roll requested 2024-12-09T06:19:17,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157170 2024-12-09T06:19:17,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157170 2024-12-09T06:19:17,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157170) roll requested 2024-12-09T06:19:17,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157176 2024-12-09T06:19:17,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157176 2024-12-09T06:19:17,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157176) roll requested 2024-12-09T06:19:17,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157183 2024-12-09T06:19:17,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157183 2024-12-09T06:19:17,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157183) roll requested 2024-12-09T06:19:17,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157191 2024-12-09T06:19:17,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157191 2024-12-09T06:19:17,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157191) roll requested 2024-12-09T06:19:17,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157200 2024-12-09T06:19:17,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157200 2024-12-09T06:19:17,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157200) roll requested 2024-12-09T06:19:17,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157210 2024-12-09T06:19:17,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157210 2024-12-09T06:19:17,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157210) roll requested 2024-12-09T06:19:17,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157220 2024-12-09T06:19:17,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157220 2024-12-09T06:19:17,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157220) roll requested 2024-12-09T06:19:17,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157226 2024-12-09T06:19:17,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157226 2024-12-09T06:19:17,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157226) roll requested 2024-12-09T06:19:17,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157233 2024-12-09T06:19:17,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157233 2024-12-09T06:19:17,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157233) roll requested 2024-12-09T06:19:17,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157241 2024-12-09T06:19:17,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157241 2024-12-09T06:19:17,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157241) roll requested 2024-12-09T06:19:17,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157247 2024-12-09T06:19:17,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157247 2024-12-09T06:19:17,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157247) roll requested 2024-12-09T06:19:17,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157254 2024-12-09T06:19:17,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157254 2024-12-09T06:19:17,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157254) roll requested 2024-12-09T06:19:17,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157261 2024-12-09T06:19:17,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157261 2024-12-09T06:19:17,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157261) roll requested 2024-12-09T06:19:17,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157318 2024-12-09T06:19:17,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157318 2024-12-09T06:19:17,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157318) roll requested 2024-12-09T06:19:17,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157326 2024-12-09T06:19:17,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157326 2024-12-09T06:19:17,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157326) roll requested 2024-12-09T06:19:17,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157332 2024-12-09T06:19:17,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157332 2024-12-09T06:19:17,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157332) roll requested 2024-12-09T06:19:17,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157339 2024-12-09T06:19:17,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157339 2024-12-09T06:19:17,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157339) roll requested 2024-12-09T06:19:17,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157349 2024-12-09T06:19:17,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157349 2024-12-09T06:19:17,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157349) roll requested 2024-12-09T06:19:17,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157357 2024-12-09T06:19:17,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157357 2024-12-09T06:19:17,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157357) roll requested 2024-12-09T06:19:17,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157364 2024-12-09T06:19:17,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157364 2024-12-09T06:19:17,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157364) roll requested 2024-12-09T06:19:17,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157370 2024-12-09T06:19:17,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157370 2024-12-09T06:19:17,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157370) roll requested 2024-12-09T06:19:17,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157377 2024-12-09T06:19:17,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157377 2024-12-09T06:19:17,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157377) roll requested 2024-12-09T06:19:17,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157385 2024-12-09T06:19:17,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157385 2024-12-09T06:19:17,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157385) roll requested 2024-12-09T06:19:17,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157393 2024-12-09T06:19:17,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157393 2024-12-09T06:19:17,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157393) roll requested 2024-12-09T06:19:17,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157399 2024-12-09T06:19:17,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157399 2024-12-09T06:19:17,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157399) roll requested 2024-12-09T06:19:17,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157405 2024-12-09T06:19:17,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157405 2024-12-09T06:19:17,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157405) roll requested 2024-12-09T06:19:17,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157410 2024-12-09T06:19:17,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157410 2024-12-09T06:19:17,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157410) roll requested 2024-12-09T06:19:17,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157416 2024-12-09T06:19:17,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157416 2024-12-09T06:19:17,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157416) roll requested 2024-12-09T06:19:17,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157421 2024-12-09T06:19:17,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157421 2024-12-09T06:19:17,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157421) roll requested 2024-12-09T06:19:17,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157426 2024-12-09T06:19:17,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157426 2024-12-09T06:19:17,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157426) roll requested 2024-12-09T06:19:17,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157431 2024-12-09T06:19:17,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157431 2024-12-09T06:19:17,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157431) roll requested 2024-12-09T06:19:17,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157436 2024-12-09T06:19:17,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157436 2024-12-09T06:19:17,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157436) roll requested 2024-12-09T06:19:17,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157441 2024-12-09T06:19:17,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157441 2024-12-09T06:19:17,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157441) roll requested 2024-12-09T06:19:17,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157446 2024-12-09T06:19:17,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157446 2024-12-09T06:19:17,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157446) roll requested 2024-12-09T06:19:17,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157451 2024-12-09T06:19:17,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157451 2024-12-09T06:19:17,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157451) roll requested 2024-12-09T06:19:17,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157459 2024-12-09T06:19:17,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157459 2024-12-09T06:19:17,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157459) roll requested 2024-12-09T06:19:17,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157467 2024-12-09T06:19:17,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157467 2024-12-09T06:19:17,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157467) roll requested 2024-12-09T06:19:17,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157472 2024-12-09T06:19:17,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157472 2024-12-09T06:19:17,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157472) roll requested 2024-12-09T06:19:17,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157477 2024-12-09T06:19:17,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157477 2024-12-09T06:19:17,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157477) roll requested 2024-12-09T06:19:17,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157482 2024-12-09T06:19:17,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157482 2024-12-09T06:19:17,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157482) roll requested 2024-12-09T06:19:17,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157488 2024-12-09T06:19:17,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157488 2024-12-09T06:19:17,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157488) roll requested 2024-12-09T06:19:17,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157493 2024-12-09T06:19:17,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157493 2024-12-09T06:19:17,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157493) roll requested 2024-12-09T06:19:17,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157499 2024-12-09T06:19:17,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157499 2024-12-09T06:19:17,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157499) roll requested 2024-12-09T06:19:17,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157505 2024-12-09T06:19:17,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157505 2024-12-09T06:19:17,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157505) roll requested 2024-12-09T06:19:17,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157510 2024-12-09T06:19:17,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157510 2024-12-09T06:19:17,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157510) roll requested 2024-12-09T06:19:17,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157516 2024-12-09T06:19:17,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157516 2024-12-09T06:19:17,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157516) roll requested 2024-12-09T06:19:17,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157521 2024-12-09T06:19:17,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157521 2024-12-09T06:19:17,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157521) roll requested 2024-12-09T06:19:17,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157529 2024-12-09T06:19:17,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157529 2024-12-09T06:19:17,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157529) roll requested 2024-12-09T06:19:17,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157535 2024-12-09T06:19:17,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157535 2024-12-09T06:19:17,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157535) roll requested 2024-12-09T06:19:17,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157541 2024-12-09T06:19:17,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157541 2024-12-09T06:19:17,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157541) roll requested 2024-12-09T06:19:17,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157549 2024-12-09T06:19:17,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157549 2024-12-09T06:19:17,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157549) roll requested 2024-12-09T06:19:17,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157554 2024-12-09T06:19:17,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157554 2024-12-09T06:19:17,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157554) roll requested 2024-12-09T06:19:17,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157560 2024-12-09T06:19:17,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157560 2024-12-09T06:19:17,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157560) roll requested 2024-12-09T06:19:17,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157566 2024-12-09T06:19:17,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157566 2024-12-09T06:19:17,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157566) roll requested 2024-12-09T06:19:17,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157571 2024-12-09T06:19:17,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157571 2024-12-09T06:19:17,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157571) roll requested 2024-12-09T06:19:17,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157576 2024-12-09T06:19:17,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157576 2024-12-09T06:19:17,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157576) roll requested 2024-12-09T06:19:17,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157581 2024-12-09T06:19:17,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157581 2024-12-09T06:19:17,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157581) roll requested 2024-12-09T06:19:17,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157586 2024-12-09T06:19:17,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157586 2024-12-09T06:19:17,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157586) roll requested 2024-12-09T06:19:17,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157590 2024-12-09T06:19:17,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157590 2024-12-09T06:19:17,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157590) roll requested 2024-12-09T06:19:17,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157595 2024-12-09T06:19:17,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157595 2024-12-09T06:19:17,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157595) roll requested 2024-12-09T06:19:17,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157602 2024-12-09T06:19:17,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157602 2024-12-09T06:19:17,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157602) roll requested 2024-12-09T06:19:17,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157608 2024-12-09T06:19:17,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157608 2024-12-09T06:19:17,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157608) roll requested 2024-12-09T06:19:17,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157616 2024-12-09T06:19:17,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157616 2024-12-09T06:19:17,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157616) roll requested 2024-12-09T06:19:17,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157624 2024-12-09T06:19:17,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157624 2024-12-09T06:19:17,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157624) roll requested 2024-12-09T06:19:17,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157634 2024-12-09T06:19:17,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157634 2024-12-09T06:19:17,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157634) roll requested 2024-12-09T06:19:17,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157640 2024-12-09T06:19:17,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157640 2024-12-09T06:19:17,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157640) roll requested 2024-12-09T06:19:17,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157645 2024-12-09T06:19:17,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157645 2024-12-09T06:19:17,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157645) roll requested 2024-12-09T06:19:17,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157652 2024-12-09T06:19:17,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157652 2024-12-09T06:19:17,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157652) roll requested 2024-12-09T06:19:17,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157658 2024-12-09T06:19:17,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157658 2024-12-09T06:19:17,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157658) roll requested 2024-12-09T06:19:17,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157664 2024-12-09T06:19:17,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157664 2024-12-09T06:19:17,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157664) roll requested 2024-12-09T06:19:17,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157671 2024-12-09T06:19:17,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157671 2024-12-09T06:19:17,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157671) roll requested 2024-12-09T06:19:17,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157678 2024-12-09T06:19:17,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157678 2024-12-09T06:19:17,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157678) roll requested 2024-12-09T06:19:17,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157685 2024-12-09T06:19:17,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157685 2024-12-09T06:19:17,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157685) roll requested 2024-12-09T06:19:17,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157694 2024-12-09T06:19:17,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157694 2024-12-09T06:19:17,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157694) roll requested 2024-12-09T06:19:17,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157704 2024-12-09T06:19:17,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157704 2024-12-09T06:19:17,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157704) roll requested 2024-12-09T06:19:17,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157712 2024-12-09T06:19:17,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157712 2024-12-09T06:19:17,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157712) roll requested 2024-12-09T06:19:17,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157721 2024-12-09T06:19:17,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157721 2024-12-09T06:19:17,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157721) roll requested 2024-12-09T06:19:17,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157730 2024-12-09T06:19:17,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157730 2024-12-09T06:19:17,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157730) roll requested 2024-12-09T06:19:17,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157735 2024-12-09T06:19:17,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157735 2024-12-09T06:19:17,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157735) roll requested 2024-12-09T06:19:17,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157739 2024-12-09T06:19:17,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157739 2024-12-09T06:19:17,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157739) roll requested 2024-12-09T06:19:17,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157744 2024-12-09T06:19:17,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157744 2024-12-09T06:19:17,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157744) roll requested 2024-12-09T06:19:17,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157749 2024-12-09T06:19:17,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157749 2024-12-09T06:19:17,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157749) roll requested 2024-12-09T06:19:17,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157756 2024-12-09T06:19:17,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157756 2024-12-09T06:19:17,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157756) roll requested 2024-12-09T06:19:17,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157763 2024-12-09T06:19:17,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157763 2024-12-09T06:19:17,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157763) roll requested 2024-12-09T06:19:17,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157770 2024-12-09T06:19:17,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157770 2024-12-09T06:19:17,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157770) roll requested 2024-12-09T06:19:17,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157778 2024-12-09T06:19:17,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157778 2024-12-09T06:19:17,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157778) roll requested 2024-12-09T06:19:17,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157785 2024-12-09T06:19:17,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157785 2024-12-09T06:19:17,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157785) roll requested 2024-12-09T06:19:17,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157794 2024-12-09T06:19:17,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157794 2024-12-09T06:19:17,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157794) roll requested 2024-12-09T06:19:17,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157803 2024-12-09T06:19:17,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157803 2024-12-09T06:19:17,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157803) roll requested 2024-12-09T06:19:17,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157813 2024-12-09T06:19:17,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157813 2024-12-09T06:19:17,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157813) roll requested 2024-12-09T06:19:17,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157818 2024-12-09T06:19:17,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157818 2024-12-09T06:19:17,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157818) roll requested 2024-12-09T06:19:17,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157829 2024-12-09T06:19:17,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157829 2024-12-09T06:19:17,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157829) roll requested 2024-12-09T06:19:17,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157834 2024-12-09T06:19:17,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157834 2024-12-09T06:19:17,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157834) roll requested 2024-12-09T06:19:17,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157840 2024-12-09T06:19:17,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157840 2024-12-09T06:19:17,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157840) roll requested 2024-12-09T06:19:17,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157845 2024-12-09T06:19:17,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157845 2024-12-09T06:19:17,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157845) roll requested 2024-12-09T06:19:17,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157853 2024-12-09T06:19:17,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157853 2024-12-09T06:19:17,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157853) roll requested 2024-12-09T06:19:17,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157860 2024-12-09T06:19:17,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157860 2024-12-09T06:19:17,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157860) roll requested 2024-12-09T06:19:17,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157865 2024-12-09T06:19:17,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157865 2024-12-09T06:19:17,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157865) roll requested 2024-12-09T06:19:17,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157873 2024-12-09T06:19:17,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157873 2024-12-09T06:19:17,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157873) roll requested 2024-12-09T06:19:17,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157881 2024-12-09T06:19:17,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157881 2024-12-09T06:19:17,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157881) roll requested 2024-12-09T06:19:17,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157886 2024-12-09T06:19:17,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157886 2024-12-09T06:19:17,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157886) roll requested 2024-12-09T06:19:17,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157892 2024-12-09T06:19:17,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157892 2024-12-09T06:19:17,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157892) roll requested 2024-12-09T06:19:17,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157898 2024-12-09T06:19:17,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157898 2024-12-09T06:19:17,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157898) roll requested 2024-12-09T06:19:17,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157905 2024-12-09T06:19:17,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157905 2024-12-09T06:19:17,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157905) roll requested 2024-12-09T06:19:17,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157913 2024-12-09T06:19:17,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157913 2024-12-09T06:19:17,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157913) roll requested 2024-12-09T06:19:17,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157921 2024-12-09T06:19:17,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157921 2024-12-09T06:19:17,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157921) roll requested 2024-12-09T06:19:17,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157927 2024-12-09T06:19:17,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157927 2024-12-09T06:19:17,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157927) roll requested 2024-12-09T06:19:17,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157933 2024-12-09T06:19:17,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157933 2024-12-09T06:19:17,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157933) roll requested 2024-12-09T06:19:17,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157937 2024-12-09T06:19:17,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157937 2024-12-09T06:19:17,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157937) roll requested 2024-12-09T06:19:17,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157942 2024-12-09T06:19:17,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157942 2024-12-09T06:19:17,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157942) roll requested 2024-12-09T06:19:17,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157947 2024-12-09T06:19:17,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157947 2024-12-09T06:19:17,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157947) roll requested 2024-12-09T06:19:17,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157953 2024-12-09T06:19:17,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157953 2024-12-09T06:19:17,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157953) roll requested 2024-12-09T06:19:17,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157959 2024-12-09T06:19:17,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157959 2024-12-09T06:19:17,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157959) roll requested 2024-12-09T06:19:17,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157968 2024-12-09T06:19:17,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157968 2024-12-09T06:19:17,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157968) roll requested 2024-12-09T06:19:17,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157976 2024-12-09T06:19:17,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157976 2024-12-09T06:19:17,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157976) roll requested 2024-12-09T06:19:17,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157980 2024-12-09T06:19:17,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157980 2024-12-09T06:19:17,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157980) roll requested 2024-12-09T06:19:17,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157986 2024-12-09T06:19:17,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157986 2024-12-09T06:19:17,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157986) roll requested 2024-12-09T06:19:17,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157991 2024-12-09T06:19:17,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:17,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157991 2024-12-09T06:19:17,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:17,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:17,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157991) roll requested 2024-12-09T06:19:17,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725157997 2024-12-09T06:19:18,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157997 2024-12-09T06:19:18,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725157997) roll requested 2024-12-09T06:19:18,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158004 2024-12-09T06:19:18,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725157997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158004 2024-12-09T06:19:18,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158004) roll requested 2024-12-09T06:19:18,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158012 2024-12-09T06:19:18,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158012 2024-12-09T06:19:18,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158012) roll requested 2024-12-09T06:19:18,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158019 2024-12-09T06:19:18,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158019 2024-12-09T06:19:18,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158019) roll requested 2024-12-09T06:19:18,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158029 2024-12-09T06:19:18,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158029 2024-12-09T06:19:18,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158029) roll requested 2024-12-09T06:19:18,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158040 2024-12-09T06:19:18,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158040 2024-12-09T06:19:18,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158040) roll requested 2024-12-09T06:19:18,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158047 2024-12-09T06:19:18,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158047 2024-12-09T06:19:18,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158047) roll requested 2024-12-09T06:19:18,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158055 2024-12-09T06:19:18,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158055 2024-12-09T06:19:18,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158055) roll requested 2024-12-09T06:19:18,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158063 2024-12-09T06:19:18,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158063 2024-12-09T06:19:18,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158063) roll requested 2024-12-09T06:19:18,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158070 2024-12-09T06:19:18,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158070 2024-12-09T06:19:18,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158070) roll requested 2024-12-09T06:19:18,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158078 2024-12-09T06:19:18,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158078 2024-12-09T06:19:18,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158078) roll requested 2024-12-09T06:19:18,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158085 2024-12-09T06:19:18,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158085 2024-12-09T06:19:18,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158085) roll requested 2024-12-09T06:19:18,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158092 2024-12-09T06:19:18,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158092 2024-12-09T06:19:18,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158092) roll requested 2024-12-09T06:19:18,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158098 2024-12-09T06:19:18,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158098 2024-12-09T06:19:18,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158098) roll requested 2024-12-09T06:19:18,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158105 2024-12-09T06:19:18,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158105 2024-12-09T06:19:18,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158105) roll requested 2024-12-09T06:19:18,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158112 2024-12-09T06:19:18,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158112 2024-12-09T06:19:18,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158112) roll requested 2024-12-09T06:19:18,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158120 2024-12-09T06:19:18,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158120 2024-12-09T06:19:18,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158120) roll requested 2024-12-09T06:19:18,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158128 2024-12-09T06:19:18,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158128 2024-12-09T06:19:18,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158128) roll requested 2024-12-09T06:19:18,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158138 2024-12-09T06:19:18,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158138 2024-12-09T06:19:18,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158138) roll requested 2024-12-09T06:19:18,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158148 2024-12-09T06:19:18,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158148 2024-12-09T06:19:18,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158148) roll requested 2024-12-09T06:19:18,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158156 2024-12-09T06:19:18,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158156 2024-12-09T06:19:18,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158156) roll requested 2024-12-09T06:19:18,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158163 2024-12-09T06:19:18,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158163 2024-12-09T06:19:18,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158163) roll requested 2024-12-09T06:19:18,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158172 2024-12-09T06:19:18,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158172 2024-12-09T06:19:18,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158172) roll requested 2024-12-09T06:19:18,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158181 2024-12-09T06:19:18,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158181 2024-12-09T06:19:18,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158181) roll requested 2024-12-09T06:19:18,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158188 2024-12-09T06:19:18,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158188 2024-12-09T06:19:18,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158188) roll requested 2024-12-09T06:19:18,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158196 2024-12-09T06:19:18,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158196 2024-12-09T06:19:18,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158196) roll requested 2024-12-09T06:19:18,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158205 2024-12-09T06:19:18,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158205 2024-12-09T06:19:18,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158205) roll requested 2024-12-09T06:19:18,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158216 2024-12-09T06:19:18,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158216 2024-12-09T06:19:18,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158216) roll requested 2024-12-09T06:19:18,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158226 2024-12-09T06:19:18,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158226 2024-12-09T06:19:18,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158226) roll requested 2024-12-09T06:19:18,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158236 2024-12-09T06:19:18,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158236 2024-12-09T06:19:18,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158236) roll requested 2024-12-09T06:19:18,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158244 2024-12-09T06:19:18,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158244 2024-12-09T06:19:18,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158244) roll requested 2024-12-09T06:19:18,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158249 2024-12-09T06:19:18,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158249 2024-12-09T06:19:18,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158249) roll requested 2024-12-09T06:19:18,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158254 2024-12-09T06:19:18,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158254 2024-12-09T06:19:18,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158254) roll requested 2024-12-09T06:19:18,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158260 2024-12-09T06:19:18,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158260 2024-12-09T06:19:18,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158260) roll requested 2024-12-09T06:19:18,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158266 2024-12-09T06:19:18,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158266 2024-12-09T06:19:18,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158266) roll requested 2024-12-09T06:19:18,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158274 2024-12-09T06:19:18,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158274 2024-12-09T06:19:18,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158274) roll requested 2024-12-09T06:19:18,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158281 2024-12-09T06:19:18,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158281 2024-12-09T06:19:18,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158281) roll requested 2024-12-09T06:19:18,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158290 2024-12-09T06:19:18,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158290 2024-12-09T06:19:18,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158290) roll requested 2024-12-09T06:19:18,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158297 2024-12-09T06:19:18,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158297 2024-12-09T06:19:18,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158297) roll requested 2024-12-09T06:19:18,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158306 2024-12-09T06:19:18,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158306 2024-12-09T06:19:18,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158306) roll requested 2024-12-09T06:19:18,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158315 2024-12-09T06:19:18,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158315 2024-12-09T06:19:18,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158315) roll requested 2024-12-09T06:19:18,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158323 2024-12-09T06:19:18,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158323 2024-12-09T06:19:18,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158323) roll requested 2024-12-09T06:19:18,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158330 2024-12-09T06:19:18,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158330 2024-12-09T06:19:18,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158330) roll requested 2024-12-09T06:19:18,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158338 2024-12-09T06:19:18,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158338 2024-12-09T06:19:18,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158338) roll requested 2024-12-09T06:19:18,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158348 2024-12-09T06:19:18,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158348 2024-12-09T06:19:18,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158348) roll requested 2024-12-09T06:19:18,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158358 2024-12-09T06:19:18,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158358 2024-12-09T06:19:18,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158358) roll requested 2024-12-09T06:19:18,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158368 2024-12-09T06:19:18,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158368 2024-12-09T06:19:18,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158368) roll requested 2024-12-09T06:19:18,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158378 2024-12-09T06:19:18,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158378 2024-12-09T06:19:18,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158378) roll requested 2024-12-09T06:19:18,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158384 2024-12-09T06:19:18,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158384 2024-12-09T06:19:18,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158384) roll requested 2024-12-09T06:19:18,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158389 2024-12-09T06:19:18,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158389 2024-12-09T06:19:18,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158389) roll requested 2024-12-09T06:19:18,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158395 2024-12-09T06:19:18,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158395 2024-12-09T06:19:18,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158395) roll requested 2024-12-09T06:19:18,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158401 2024-12-09T06:19:18,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158401 2024-12-09T06:19:18,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158401) roll requested 2024-12-09T06:19:18,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158409 2024-12-09T06:19:18,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158409 2024-12-09T06:19:18,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158409) roll requested 2024-12-09T06:19:18,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158417 2024-12-09T06:19:18,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158417 2024-12-09T06:19:18,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158417) roll requested 2024-12-09T06:19:18,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158424 2024-12-09T06:19:18,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158424 2024-12-09T06:19:18,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158424) roll requested 2024-12-09T06:19:18,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158432 2024-12-09T06:19:18,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158432 2024-12-09T06:19:18,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158432) roll requested 2024-12-09T06:19:18,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158441 2024-12-09T06:19:18,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158441 2024-12-09T06:19:18,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158441) roll requested 2024-12-09T06:19:18,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158453 2024-12-09T06:19:18,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158453 2024-12-09T06:19:18,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158453) roll requested 2024-12-09T06:19:18,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158458 2024-12-09T06:19:18,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158458 2024-12-09T06:19:18,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158458) roll requested 2024-12-09T06:19:18,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158465 2024-12-09T06:19:18,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158465 2024-12-09T06:19:18,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158465) roll requested 2024-12-09T06:19:18,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158470 2024-12-09T06:19:18,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158470 2024-12-09T06:19:18,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158470) roll requested 2024-12-09T06:19:18,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158475 2024-12-09T06:19:18,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158475 2024-12-09T06:19:18,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158475) roll requested 2024-12-09T06:19:18,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158481 2024-12-09T06:19:18,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158481 2024-12-09T06:19:18,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158481) roll requested 2024-12-09T06:19:18,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158485 2024-12-09T06:19:18,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158485 2024-12-09T06:19:18,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158485) roll requested 2024-12-09T06:19:18,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158490 2024-12-09T06:19:18,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158490 2024-12-09T06:19:18,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158490) roll requested 2024-12-09T06:19:18,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158495 2024-12-09T06:19:18,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158495 2024-12-09T06:19:18,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158495) roll requested 2024-12-09T06:19:18,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158502 2024-12-09T06:19:18,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158502 2024-12-09T06:19:18,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158502) roll requested 2024-12-09T06:19:18,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158509 2024-12-09T06:19:18,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158509 2024-12-09T06:19:18,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158509) roll requested 2024-12-09T06:19:18,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158517 2024-12-09T06:19:18,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158517 2024-12-09T06:19:18,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158517) roll requested 2024-12-09T06:19:18,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158526 2024-12-09T06:19:18,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158526 2024-12-09T06:19:18,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158526) roll requested 2024-12-09T06:19:18,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158534 2024-12-09T06:19:18,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158534 2024-12-09T06:19:18,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158534) roll requested 2024-12-09T06:19:18,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158544 2024-12-09T06:19:18,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158544 2024-12-09T06:19:18,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158544) roll requested 2024-12-09T06:19:18,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158553 2024-12-09T06:19:18,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158553 2024-12-09T06:19:18,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158553) roll requested 2024-12-09T06:19:18,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158561 2024-12-09T06:19:18,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158561 2024-12-09T06:19:18,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158561) roll requested 2024-12-09T06:19:18,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158570 2024-12-09T06:19:18,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158570 2024-12-09T06:19:18,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158570) roll requested 2024-12-09T06:19:18,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158578 2024-12-09T06:19:18,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158578 2024-12-09T06:19:18,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158578) roll requested 2024-12-09T06:19:18,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158588 2024-12-09T06:19:18,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158588 2024-12-09T06:19:18,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158588) roll requested 2024-12-09T06:19:18,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158595 2024-12-09T06:19:18,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158595 2024-12-09T06:19:18,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158595) roll requested 2024-12-09T06:19:18,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158601 2024-12-09T06:19:18,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158601 2024-12-09T06:19:18,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158601) roll requested 2024-12-09T06:19:18,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158607 2024-12-09T06:19:18,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158607 2024-12-09T06:19:18,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158607) roll requested 2024-12-09T06:19:18,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158615 2024-12-09T06:19:18,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158615 2024-12-09T06:19:18,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158615) roll requested 2024-12-09T06:19:18,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158623 2024-12-09T06:19:18,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158623 2024-12-09T06:19:18,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158623) roll requested 2024-12-09T06:19:18,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158632 2024-12-09T06:19:18,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158632 2024-12-09T06:19:18,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158632) roll requested 2024-12-09T06:19:18,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158641 2024-12-09T06:19:18,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158641 2024-12-09T06:19:18,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158641) roll requested 2024-12-09T06:19:18,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158646 2024-12-09T06:19:18,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158646 2024-12-09T06:19:18,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158646) roll requested 2024-12-09T06:19:18,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158652 2024-12-09T06:19:18,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158652 2024-12-09T06:19:18,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158652) roll requested 2024-12-09T06:19:18,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158659 2024-12-09T06:19:18,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158659 2024-12-09T06:19:18,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158659) roll requested 2024-12-09T06:19:18,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158667 2024-12-09T06:19:18,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158667 2024-12-09T06:19:18,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158667) roll requested 2024-12-09T06:19:18,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158675 2024-12-09T06:19:18,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158675 2024-12-09T06:19:18,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158675) roll requested 2024-12-09T06:19:18,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158684 2024-12-09T06:19:18,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158684 2024-12-09T06:19:18,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158684) roll requested 2024-12-09T06:19:18,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158693 2024-12-09T06:19:18,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158693 2024-12-09T06:19:18,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158693) roll requested 2024-12-09T06:19:18,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158702 2024-12-09T06:19:18,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158702 2024-12-09T06:19:18,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158702) roll requested 2024-12-09T06:19:18,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158711 2024-12-09T06:19:18,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158711 2024-12-09T06:19:18,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158711) roll requested 2024-12-09T06:19:18,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158720 2024-12-09T06:19:18,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158720 2024-12-09T06:19:18,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158720) roll requested 2024-12-09T06:19:18,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158727 2024-12-09T06:19:18,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158727 2024-12-09T06:19:18,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158727) roll requested 2024-12-09T06:19:18,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158734 2024-12-09T06:19:18,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158734 2024-12-09T06:19:18,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158734) roll requested 2024-12-09T06:19:18,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158742 2024-12-09T06:19:18,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158742 2024-12-09T06:19:18,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158742) roll requested 2024-12-09T06:19:18,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158751 2024-12-09T06:19:18,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158751 2024-12-09T06:19:18,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158751) roll requested 2024-12-09T06:19:18,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158759 2024-12-09T06:19:18,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158759 2024-12-09T06:19:18,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158759) roll requested 2024-12-09T06:19:18,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158766 2024-12-09T06:19:18,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158766 2024-12-09T06:19:18,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158766) roll requested 2024-12-09T06:19:18,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158775 2024-12-09T06:19:18,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158775 2024-12-09T06:19:18,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158775) roll requested 2024-12-09T06:19:18,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158785 2024-12-09T06:19:18,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158785 2024-12-09T06:19:18,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158785) roll requested 2024-12-09T06:19:18,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158792 2024-12-09T06:19:18,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158792 2024-12-09T06:19:18,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158792) roll requested 2024-12-09T06:19:18,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158797 2024-12-09T06:19:18,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158797 2024-12-09T06:19:18,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158797) roll requested 2024-12-09T06:19:18,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158803 2024-12-09T06:19:18,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158803 2024-12-09T06:19:18,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158803) roll requested 2024-12-09T06:19:18,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158809 2024-12-09T06:19:18,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158809 2024-12-09T06:19:18,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158809) roll requested 2024-12-09T06:19:18,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158817 2024-12-09T06:19:18,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158817 2024-12-09T06:19:18,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158817) roll requested 2024-12-09T06:19:18,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158829 2024-12-09T06:19:18,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158829 2024-12-09T06:19:18,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158829) roll requested 2024-12-09T06:19:18,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158834 2024-12-09T06:19:18,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158834 2024-12-09T06:19:18,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158834) roll requested 2024-12-09T06:19:18,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158839 2024-12-09T06:19:18,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158839 2024-12-09T06:19:18,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158839) roll requested 2024-12-09T06:19:18,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158845 2024-12-09T06:19:18,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158845 2024-12-09T06:19:18,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158845) roll requested 2024-12-09T06:19:18,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158851 2024-12-09T06:19:18,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158851 2024-12-09T06:19:18,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158851) roll requested 2024-12-09T06:19:18,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158856 2024-12-09T06:19:18,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158856 2024-12-09T06:19:18,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158856) roll requested 2024-12-09T06:19:18,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158862 2024-12-09T06:19:18,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158862 2024-12-09T06:19:18,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158862) roll requested 2024-12-09T06:19:18,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158867 2024-12-09T06:19:18,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158867 2024-12-09T06:19:18,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158867) roll requested 2024-12-09T06:19:18,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158871 2024-12-09T06:19:18,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158871 2024-12-09T06:19:18,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158871) roll requested 2024-12-09T06:19:18,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158875 2024-12-09T06:19:18,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158875 2024-12-09T06:19:18,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158875) roll requested 2024-12-09T06:19:18,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158882 2024-12-09T06:19:18,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158882 2024-12-09T06:19:18,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158882) roll requested 2024-12-09T06:19:18,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158888 2024-12-09T06:19:18,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158888 2024-12-09T06:19:18,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158888) roll requested 2024-12-09T06:19:18,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158895 2024-12-09T06:19:18,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158895 2024-12-09T06:19:18,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158895) roll requested 2024-12-09T06:19:18,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158903 2024-12-09T06:19:18,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158903 2024-12-09T06:19:18,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158903) roll requested 2024-12-09T06:19:18,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158910 2024-12-09T06:19:18,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158910 2024-12-09T06:19:18,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158910) roll requested 2024-12-09T06:19:18,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158919 2024-12-09T06:19:18,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158919 2024-12-09T06:19:18,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158919) roll requested 2024-12-09T06:19:18,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158928 2024-12-09T06:19:18,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158928 2024-12-09T06:19:18,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158928) roll requested 2024-12-09T06:19:18,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158935 2024-12-09T06:19:18,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158935 2024-12-09T06:19:18,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158935) roll requested 2024-12-09T06:19:18,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158941 2024-12-09T06:19:18,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158941 2024-12-09T06:19:18,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158941) roll requested 2024-12-09T06:19:18,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158948 2024-12-09T06:19:18,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158948 2024-12-09T06:19:18,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158948) roll requested 2024-12-09T06:19:18,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158956 2024-12-09T06:19:18,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158956 2024-12-09T06:19:18,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158956) roll requested 2024-12-09T06:19:18,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158962 2024-12-09T06:19:18,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158962 2024-12-09T06:19:18,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158962) roll requested 2024-12-09T06:19:18,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158970 2024-12-09T06:19:18,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158970 2024-12-09T06:19:18,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158970) roll requested 2024-12-09T06:19:18,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158978 2024-12-09T06:19:18,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158978 2024-12-09T06:19:18,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158978) roll requested 2024-12-09T06:19:18,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158986 2024-12-09T06:19:18,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:18,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158986 2024-12-09T06:19:18,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:18,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:18,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158986) roll requested 2024-12-09T06:19:18,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725158996 2024-12-09T06:19:19,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158996 2024-12-09T06:19:19,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725158996) roll requested 2024-12-09T06:19:19,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159003 2024-12-09T06:19:19,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725158996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159003 2024-12-09T06:19:19,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159003) roll requested 2024-12-09T06:19:19,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159008 2024-12-09T06:19:19,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159008 2024-12-09T06:19:19,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159008) roll requested 2024-12-09T06:19:19,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159015 2024-12-09T06:19:19,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159015 2024-12-09T06:19:19,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159015) roll requested 2024-12-09T06:19:19,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159022 2024-12-09T06:19:19,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159022 2024-12-09T06:19:19,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159022) roll requested 2024-12-09T06:19:19,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159029 2024-12-09T06:19:19,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159029 2024-12-09T06:19:19,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159029) roll requested 2024-12-09T06:19:19,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159035 2024-12-09T06:19:19,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159035 2024-12-09T06:19:19,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159035) roll requested 2024-12-09T06:19:19,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159043 2024-12-09T06:19:19,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159043 2024-12-09T06:19:19,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159043) roll requested 2024-12-09T06:19:19,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159051 2024-12-09T06:19:19,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159051 2024-12-09T06:19:19,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159051) roll requested 2024-12-09T06:19:19,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159059 2024-12-09T06:19:19,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159059 2024-12-09T06:19:19,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159059) roll requested 2024-12-09T06:19:19,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159068 2024-12-09T06:19:19,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159068 2024-12-09T06:19:19,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159068) roll requested 2024-12-09T06:19:19,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159076 2024-12-09T06:19:19,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159076 2024-12-09T06:19:19,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159076) roll requested 2024-12-09T06:19:19,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159083 2024-12-09T06:19:19,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159083 2024-12-09T06:19:19,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159083) roll requested 2024-12-09T06:19:19,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159091 2024-12-09T06:19:19,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159091 2024-12-09T06:19:19,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159091) roll requested 2024-12-09T06:19:19,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159099 2024-12-09T06:19:19,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159099 2024-12-09T06:19:19,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159099) roll requested 2024-12-09T06:19:19,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159107 2024-12-09T06:19:19,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159107 2024-12-09T06:19:19,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159107) roll requested 2024-12-09T06:19:19,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159115 2024-12-09T06:19:19,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159115 2024-12-09T06:19:19,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159115) roll requested 2024-12-09T06:19:19,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159123 2024-12-09T06:19:19,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159123 2024-12-09T06:19:19,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159123) roll requested 2024-12-09T06:19:19,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159129 2024-12-09T06:19:19,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159129 2024-12-09T06:19:19,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159129) roll requested 2024-12-09T06:19:19,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159136 2024-12-09T06:19:19,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159136 2024-12-09T06:19:19,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159136) roll requested 2024-12-09T06:19:19,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159143 2024-12-09T06:19:19,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159143 2024-12-09T06:19:19,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159143) roll requested 2024-12-09T06:19:19,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159152 2024-12-09T06:19:19,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159152 2024-12-09T06:19:19,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159152) roll requested 2024-12-09T06:19:19,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159159 2024-12-09T06:19:19,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159159 2024-12-09T06:19:19,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159159) roll requested 2024-12-09T06:19:19,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159167 2024-12-09T06:19:19,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159167 2024-12-09T06:19:19,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159167) roll requested 2024-12-09T06:19:19,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159173 2024-12-09T06:19:19,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159173 2024-12-09T06:19:19,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159173) roll requested 2024-12-09T06:19:19,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159179 2024-12-09T06:19:19,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159179 2024-12-09T06:19:19,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159179) roll requested 2024-12-09T06:19:19,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159186 2024-12-09T06:19:19,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159186 2024-12-09T06:19:19,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159186) roll requested 2024-12-09T06:19:19,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159195 2024-12-09T06:19:19,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159195 2024-12-09T06:19:19,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159195) roll requested 2024-12-09T06:19:19,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159204 2024-12-09T06:19:19,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159204 2024-12-09T06:19:19,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159204) roll requested 2024-12-09T06:19:19,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159213 2024-12-09T06:19:19,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159213 2024-12-09T06:19:19,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159213) roll requested 2024-12-09T06:19:19,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159219 2024-12-09T06:19:19,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159219 2024-12-09T06:19:19,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159219) roll requested 2024-12-09T06:19:19,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159227 2024-12-09T06:19:19,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159227 2024-12-09T06:19:19,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159227) roll requested 2024-12-09T06:19:19,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159235 2024-12-09T06:19:19,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159235 2024-12-09T06:19:19,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159235) roll requested 2024-12-09T06:19:19,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159244 2024-12-09T06:19:19,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159244 2024-12-09T06:19:19,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159244) roll requested 2024-12-09T06:19:19,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159253 2024-12-09T06:19:19,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159253 2024-12-09T06:19:19,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159253) roll requested 2024-12-09T06:19:19,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159258 2024-12-09T06:19:19,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159258 2024-12-09T06:19:19,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159258) roll requested 2024-12-09T06:19:19,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159265 2024-12-09T06:19:19,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159265 2024-12-09T06:19:19,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159265) roll requested 2024-12-09T06:19:19,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159272 2024-12-09T06:19:19,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159272 2024-12-09T06:19:19,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159272) roll requested 2024-12-09T06:19:19,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159280 2024-12-09T06:19:19,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159280 2024-12-09T06:19:19,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159280) roll requested 2024-12-09T06:19:19,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159289 2024-12-09T06:19:19,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159289 2024-12-09T06:19:19,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159289) roll requested 2024-12-09T06:19:19,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159298 2024-12-09T06:19:19,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159298 2024-12-09T06:19:19,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159298) roll requested 2024-12-09T06:19:19,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159307 2024-12-09T06:19:19,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159307 2024-12-09T06:19:19,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159307) roll requested 2024-12-09T06:19:19,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159313 2024-12-09T06:19:19,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159313 2024-12-09T06:19:19,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159313) roll requested 2024-12-09T06:19:19,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159320 2024-12-09T06:19:19,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159320 2024-12-09T06:19:19,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159320) roll requested 2024-12-09T06:19:19,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159327 2024-12-09T06:19:19,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159327 2024-12-09T06:19:19,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159327) roll requested 2024-12-09T06:19:19,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159334 2024-12-09T06:19:19,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159334 2024-12-09T06:19:19,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159334) roll requested 2024-12-09T06:19:19,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159341 2024-12-09T06:19:19,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159341 2024-12-09T06:19:19,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159341) roll requested 2024-12-09T06:19:19,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159348 2024-12-09T06:19:19,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159348 2024-12-09T06:19:19,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159348) roll requested 2024-12-09T06:19:19,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159356 2024-12-09T06:19:19,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159356 2024-12-09T06:19:19,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159356) roll requested 2024-12-09T06:19:19,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159364 2024-12-09T06:19:19,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159364 2024-12-09T06:19:19,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159364) roll requested 2024-12-09T06:19:19,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159372 2024-12-09T06:19:19,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159372 2024-12-09T06:19:19,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159372) roll requested 2024-12-09T06:19:19,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159379 2024-12-09T06:19:19,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159379 2024-12-09T06:19:19,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159379) roll requested 2024-12-09T06:19:19,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159386 2024-12-09T06:19:19,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159386 2024-12-09T06:19:19,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159386) roll requested 2024-12-09T06:19:19,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159394 2024-12-09T06:19:19,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159394 2024-12-09T06:19:19,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159394) roll requested 2024-12-09T06:19:19,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159403 2024-12-09T06:19:19,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159403 2024-12-09T06:19:19,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159403) roll requested 2024-12-09T06:19:19,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159412 2024-12-09T06:19:19,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159412 2024-12-09T06:19:19,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159412) roll requested 2024-12-09T06:19:19,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159420 2024-12-09T06:19:19,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159420 2024-12-09T06:19:19,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159420) roll requested 2024-12-09T06:19:19,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159429 2024-12-09T06:19:19,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159429 2024-12-09T06:19:19,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159429) roll requested 2024-12-09T06:19:19,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159438 2024-12-09T06:19:19,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159438 2024-12-09T06:19:19,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159438) roll requested 2024-12-09T06:19:19,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159448 2024-12-09T06:19:19,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159448 2024-12-09T06:19:19,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159448) roll requested 2024-12-09T06:19:19,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159456 2024-12-09T06:19:19,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159456 2024-12-09T06:19:19,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159456) roll requested 2024-12-09T06:19:19,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159464 2024-12-09T06:19:19,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159464 2024-12-09T06:19:19,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159464) roll requested 2024-12-09T06:19:19,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159471 2024-12-09T06:19:19,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159471 2024-12-09T06:19:19,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159471) roll requested 2024-12-09T06:19:19,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159477 2024-12-09T06:19:19,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159477 2024-12-09T06:19:19,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159477) roll requested 2024-12-09T06:19:19,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159485 2024-12-09T06:19:19,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159485 2024-12-09T06:19:19,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159485) roll requested 2024-12-09T06:19:19,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159493 2024-12-09T06:19:19,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159493 2024-12-09T06:19:19,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159493) roll requested 2024-12-09T06:19:19,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159499 2024-12-09T06:19:19,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159499 2024-12-09T06:19:19,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159499) roll requested 2024-12-09T06:19:19,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159506 2024-12-09T06:19:19,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159506 2024-12-09T06:19:19,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159506) roll requested 2024-12-09T06:19:19,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159515 2024-12-09T06:19:19,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159515 2024-12-09T06:19:19,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159515) roll requested 2024-12-09T06:19:19,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159523 2024-12-09T06:19:19,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159523 2024-12-09T06:19:19,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159523) roll requested 2024-12-09T06:19:19,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159530 2024-12-09T06:19:19,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159530 2024-12-09T06:19:19,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159530) roll requested 2024-12-09T06:19:19,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159539 2024-12-09T06:19:19,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159539 2024-12-09T06:19:19,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159539) roll requested 2024-12-09T06:19:19,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159548 2024-12-09T06:19:19,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159548 2024-12-09T06:19:19,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159548) roll requested 2024-12-09T06:19:19,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159556 2024-12-09T06:19:19,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159556 2024-12-09T06:19:19,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159556) roll requested 2024-12-09T06:19:19,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159566 2024-12-09T06:19:19,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159566 2024-12-09T06:19:19,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159566) roll requested 2024-12-09T06:19:19,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159574 2024-12-09T06:19:19,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159574 2024-12-09T06:19:19,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159574) roll requested 2024-12-09T06:19:19,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159585 2024-12-09T06:19:19,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159585 2024-12-09T06:19:19,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159585) roll requested 2024-12-09T06:19:19,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159591 2024-12-09T06:19:19,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159591 2024-12-09T06:19:19,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159591) roll requested 2024-12-09T06:19:19,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159597 2024-12-09T06:19:19,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159597 2024-12-09T06:19:19,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159597) roll requested 2024-12-09T06:19:19,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159604 2024-12-09T06:19:19,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159604 2024-12-09T06:19:19,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159604) roll requested 2024-12-09T06:19:19,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159612 2024-12-09T06:19:19,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159612 2024-12-09T06:19:19,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159612) roll requested 2024-12-09T06:19:19,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159621 2024-12-09T06:19:19,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159621 2024-12-09T06:19:19,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159621) roll requested 2024-12-09T06:19:19,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159629 2024-12-09T06:19:19,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159629 2024-12-09T06:19:19,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159629) roll requested 2024-12-09T06:19:19,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159638 2024-12-09T06:19:19,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159638 2024-12-09T06:19:19,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159638) roll requested 2024-12-09T06:19:19,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159648 2024-12-09T06:19:19,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159648 2024-12-09T06:19:19,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159648) roll requested 2024-12-09T06:19:19,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159657 2024-12-09T06:19:19,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159657 2024-12-09T06:19:19,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159657) roll requested 2024-12-09T06:19:19,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159666 2024-12-09T06:19:19,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159666 2024-12-09T06:19:19,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159666) roll requested 2024-12-09T06:19:19,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159673 2024-12-09T06:19:19,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159673 2024-12-09T06:19:19,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159673) roll requested 2024-12-09T06:19:19,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159682 2024-12-09T06:19:19,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159682 2024-12-09T06:19:19,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159682) roll requested 2024-12-09T06:19:19,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159689 2024-12-09T06:19:19,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159689 2024-12-09T06:19:19,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159689) roll requested 2024-12-09T06:19:19,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159696 2024-12-09T06:19:19,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159696 2024-12-09T06:19:19,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159696) roll requested 2024-12-09T06:19:19,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159704 2024-12-09T06:19:19,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159704 2024-12-09T06:19:19,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159704) roll requested 2024-12-09T06:19:19,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159711 2024-12-09T06:19:19,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159711 2024-12-09T06:19:19,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159711) roll requested 2024-12-09T06:19:19,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159723 2024-12-09T06:19:19,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159723 2024-12-09T06:19:19,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159723) roll requested 2024-12-09T06:19:19,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159727 2024-12-09T06:19:19,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159727 2024-12-09T06:19:19,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159727) roll requested 2024-12-09T06:19:19,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159733 2024-12-09T06:19:19,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159733 2024-12-09T06:19:19,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159733) roll requested 2024-12-09T06:19:19,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159739 2024-12-09T06:19:19,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159739 2024-12-09T06:19:19,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159739) roll requested 2024-12-09T06:19:19,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159746 2024-12-09T06:19:19,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159746 2024-12-09T06:19:19,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159746) roll requested 2024-12-09T06:19:19,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159753 2024-12-09T06:19:19,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159753 2024-12-09T06:19:19,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159753) roll requested 2024-12-09T06:19:19,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159762 2024-12-09T06:19:19,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159762 2024-12-09T06:19:19,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159762) roll requested 2024-12-09T06:19:19,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159769 2024-12-09T06:19:19,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159769 2024-12-09T06:19:19,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159769) roll requested 2024-12-09T06:19:19,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159777 2024-12-09T06:19:19,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159777 2024-12-09T06:19:19,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159777) roll requested 2024-12-09T06:19:19,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159786 2024-12-09T06:19:19,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159786 2024-12-09T06:19:19,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159786) roll requested 2024-12-09T06:19:19,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159794 2024-12-09T06:19:19,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159794 2024-12-09T06:19:19,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159794) roll requested 2024-12-09T06:19:19,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159803 2024-12-09T06:19:19,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159803 2024-12-09T06:19:19,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159803) roll requested 2024-12-09T06:19:19,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159813 2024-12-09T06:19:19,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159813 2024-12-09T06:19:19,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159813) roll requested 2024-12-09T06:19:19,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159822 2024-12-09T06:19:19,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159822 2024-12-09T06:19:19,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159822) roll requested 2024-12-09T06:19:19,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159831 2024-12-09T06:19:19,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159831 2024-12-09T06:19:19,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159831) roll requested 2024-12-09T06:19:19,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159837 2024-12-09T06:19:19,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159837 2024-12-09T06:19:19,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159837) roll requested 2024-12-09T06:19:19,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159844 2024-12-09T06:19:19,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159844 2024-12-09T06:19:19,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159844) roll requested 2024-12-09T06:19:19,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159850 2024-12-09T06:19:19,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159850 2024-12-09T06:19:19,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159850) roll requested 2024-12-09T06:19:19,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159858 2024-12-09T06:19:19,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159858 2024-12-09T06:19:19,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159858) roll requested 2024-12-09T06:19:19,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159867 2024-12-09T06:19:19,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159867 2024-12-09T06:19:19,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159867) roll requested 2024-12-09T06:19:19,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159875 2024-12-09T06:19:19,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159875 2024-12-09T06:19:19,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159875) roll requested 2024-12-09T06:19:19,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159883 2024-12-09T06:19:19,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159883 2024-12-09T06:19:19,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159883) roll requested 2024-12-09T06:19:19,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159891 2024-12-09T06:19:19,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159891 2024-12-09T06:19:19,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159891) roll requested 2024-12-09T06:19:19,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159901 2024-12-09T06:19:19,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159901 2024-12-09T06:19:19,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159901) roll requested 2024-12-09T06:19:19,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159911 2024-12-09T06:19:19,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159911 2024-12-09T06:19:19,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159911) roll requested 2024-12-09T06:19:19,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159919 2024-12-09T06:19:19,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159919 2024-12-09T06:19:19,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159919) roll requested 2024-12-09T06:19:19,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159926 2024-12-09T06:19:19,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159926 2024-12-09T06:19:19,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159926) roll requested 2024-12-09T06:19:19,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159937 2024-12-09T06:19:19,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159937 2024-12-09T06:19:19,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159937) roll requested 2024-12-09T06:19:19,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159943 2024-12-09T06:19:19,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159943 2024-12-09T06:19:19,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159943) roll requested 2024-12-09T06:19:19,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159948 2024-12-09T06:19:19,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159948 2024-12-09T06:19:19,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159948) roll requested 2024-12-09T06:19:19,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159952 2024-12-09T06:19:19,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159952 2024-12-09T06:19:19,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159952) roll requested 2024-12-09T06:19:19,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159957 2024-12-09T06:19:19,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159957 2024-12-09T06:19:19,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159957) roll requested 2024-12-09T06:19:19,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159964 2024-12-09T06:19:19,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159964 2024-12-09T06:19:19,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159964) roll requested 2024-12-09T06:19:19,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159971 2024-12-09T06:19:19,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159971 2024-12-09T06:19:19,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159971) roll requested 2024-12-09T06:19:19,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159977 2024-12-09T06:19:19,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159977 2024-12-09T06:19:19,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159977) roll requested 2024-12-09T06:19:19,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159983 2024-12-09T06:19:19,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159983 2024-12-09T06:19:19,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:19,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:19,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159983) roll requested 2024-12-09T06:19:19,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725159991 2024-12-09T06:19:19,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:19,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159991 2024-12-09T06:19:20,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725159991) roll requested 2024-12-09T06:19:20,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160000 2024-12-09T06:19:20,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725159991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160000 2024-12-09T06:19:20,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160000) roll requested 2024-12-09T06:19:20,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160008 2024-12-09T06:19:20,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160008 2024-12-09T06:19:20,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160008) roll requested 2024-12-09T06:19:20,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160016 2024-12-09T06:19:20,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160016 2024-12-09T06:19:20,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160016) roll requested 2024-12-09T06:19:20,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160024 2024-12-09T06:19:20,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160024 2024-12-09T06:19:20,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160024) roll requested 2024-12-09T06:19:20,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160030 2024-12-09T06:19:20,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160030 2024-12-09T06:19:20,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160030) roll requested 2024-12-09T06:19:20,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160037 2024-12-09T06:19:20,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160037 2024-12-09T06:19:20,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160037) roll requested 2024-12-09T06:19:20,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160045 2024-12-09T06:19:20,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160045 2024-12-09T06:19:20,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160045) roll requested 2024-12-09T06:19:20,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160054 2024-12-09T06:19:20,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160054 2024-12-09T06:19:20,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160054) roll requested 2024-12-09T06:19:20,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160061 2024-12-09T06:19:20,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160061 2024-12-09T06:19:20,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160061) roll requested 2024-12-09T06:19:20,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160072 2024-12-09T06:19:20,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160072 2024-12-09T06:19:20,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160072) roll requested 2024-12-09T06:19:20,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160078 2024-12-09T06:19:20,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160078 2024-12-09T06:19:20,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160078) roll requested 2024-12-09T06:19:20,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160083 2024-12-09T06:19:20,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160083 2024-12-09T06:19:20,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160083) roll requested 2024-12-09T06:19:20,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160089 2024-12-09T06:19:20,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160089 2024-12-09T06:19:20,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160089) roll requested 2024-12-09T06:19:20,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160095 2024-12-09T06:19:20,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160095 2024-12-09T06:19:20,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160095) roll requested 2024-12-09T06:19:20,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160102 2024-12-09T06:19:20,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160102 2024-12-09T06:19:20,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160102) roll requested 2024-12-09T06:19:20,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160109 2024-12-09T06:19:20,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160109 2024-12-09T06:19:20,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160109) roll requested 2024-12-09T06:19:20,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160117 2024-12-09T06:19:20,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160117 2024-12-09T06:19:20,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160117) roll requested 2024-12-09T06:19:20,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160127 2024-12-09T06:19:20,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160127 2024-12-09T06:19:20,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160127) roll requested 2024-12-09T06:19:20,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160138 2024-12-09T06:19:20,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160138 2024-12-09T06:19:20,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160138) roll requested 2024-12-09T06:19:20,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160148 2024-12-09T06:19:20,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160148 2024-12-09T06:19:20,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160148) roll requested 2024-12-09T06:19:20,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160155 2024-12-09T06:19:20,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160155 2024-12-09T06:19:20,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160155) roll requested 2024-12-09T06:19:20,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160160 2024-12-09T06:19:20,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160160 2024-12-09T06:19:20,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160160) roll requested 2024-12-09T06:19:20,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160165 2024-12-09T06:19:20,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160165 2024-12-09T06:19:20,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160165) roll requested 2024-12-09T06:19:20,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160171 2024-12-09T06:19:20,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160171 2024-12-09T06:19:20,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160171) roll requested 2024-12-09T06:19:20,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160177 2024-12-09T06:19:20,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160177 2024-12-09T06:19:20,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160177) roll requested 2024-12-09T06:19:20,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160184 2024-12-09T06:19:20,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160184 2024-12-09T06:19:20,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160184) roll requested 2024-12-09T06:19:20,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160191 2024-12-09T06:19:20,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160191 2024-12-09T06:19:20,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160191) roll requested 2024-12-09T06:19:20,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160199 2024-12-09T06:19:20,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160199 2024-12-09T06:19:20,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160199) roll requested 2024-12-09T06:19:20,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160209 2024-12-09T06:19:20,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160209 2024-12-09T06:19:20,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160209) roll requested 2024-12-09T06:19:20,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160218 2024-12-09T06:19:20,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160218 2024-12-09T06:19:20,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160218) roll requested 2024-12-09T06:19:20,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160225 2024-12-09T06:19:20,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160225 2024-12-09T06:19:20,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160225) roll requested 2024-12-09T06:19:20,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160231 2024-12-09T06:19:20,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160231 2024-12-09T06:19:20,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160231) roll requested 2024-12-09T06:19:20,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160236 2024-12-09T06:19:20,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160236 2024-12-09T06:19:20,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160236) roll requested 2024-12-09T06:19:20,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160241 2024-12-09T06:19:20,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160241 2024-12-09T06:19:20,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160241) roll requested 2024-12-09T06:19:20,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160246 2024-12-09T06:19:20,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160246 2024-12-09T06:19:20,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160246) roll requested 2024-12-09T06:19:20,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160251 2024-12-09T06:19:20,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160251 2024-12-09T06:19:20,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160251) roll requested 2024-12-09T06:19:20,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160258 2024-12-09T06:19:20,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160258 2024-12-09T06:19:20,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160258) roll requested 2024-12-09T06:19:20,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160265 2024-12-09T06:19:20,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160265 2024-12-09T06:19:20,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160265) roll requested 2024-12-09T06:19:20,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160274 2024-12-09T06:19:20,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160274 2024-12-09T06:19:20,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160274) roll requested 2024-12-09T06:19:20,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160281 2024-12-09T06:19:20,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160281 2024-12-09T06:19:20,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160281) roll requested 2024-12-09T06:19:20,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160290 2024-12-09T06:19:20,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160290 2024-12-09T06:19:20,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160290) roll requested 2024-12-09T06:19:20,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160299 2024-12-09T06:19:20,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160299 2024-12-09T06:19:20,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160299) roll requested 2024-12-09T06:19:20,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160306 2024-12-09T06:19:20,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160306 2024-12-09T06:19:20,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:20,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160306) roll requested 2024-12-09T06:19:20,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160319 2024-12-09T06:19:20,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160319 2024-12-09T06:19:20,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160319) roll requested 2024-12-09T06:19:20,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160326 2024-12-09T06:19:20,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160326 2024-12-09T06:19:20,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160326) roll requested 2024-12-09T06:19:20,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160331 2024-12-09T06:19:20,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160331 2024-12-09T06:19:20,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160331) roll requested 2024-12-09T06:19:20,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160341 2024-12-09T06:19:20,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160341 2024-12-09T06:19:20,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160341) roll requested 2024-12-09T06:19:20,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160353 2024-12-09T06:19:20,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160353 2024-12-09T06:19:20,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160353) roll requested 2024-12-09T06:19:20,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160359 2024-12-09T06:19:20,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160359 2024-12-09T06:19:20,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160359) roll requested 2024-12-09T06:19:20,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160364 2024-12-09T06:19:20,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160364 2024-12-09T06:19:20,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160364) roll requested 2024-12-09T06:19:20,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160369 2024-12-09T06:19:20,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160369 2024-12-09T06:19:20,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160369) roll requested 2024-12-09T06:19:20,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160377 2024-12-09T06:19:20,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160377 2024-12-09T06:19:20,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160377) roll requested 2024-12-09T06:19:20,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160384 2024-12-09T06:19:20,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160384 2024-12-09T06:19:20,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160384) roll requested 2024-12-09T06:19:20,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160391 2024-12-09T06:19:20,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160391 2024-12-09T06:19:20,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160391) roll requested 2024-12-09T06:19:20,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160402 2024-12-09T06:19:20,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160402 2024-12-09T06:19:20,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160402) roll requested 2024-12-09T06:19:20,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160407 2024-12-09T06:19:20,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160407 2024-12-09T06:19:20,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160407) roll requested 2024-12-09T06:19:20,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160413 2024-12-09T06:19:20,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160413 2024-12-09T06:19:20,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160413) roll requested 2024-12-09T06:19:20,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160419 2024-12-09T06:19:20,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160419 2024-12-09T06:19:20,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160419) roll requested 2024-12-09T06:19:20,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160426 2024-12-09T06:19:20,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160426 2024-12-09T06:19:20,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160426) roll requested 2024-12-09T06:19:20,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160435 2024-12-09T06:19:20,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160435 2024-12-09T06:19:20,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160435) roll requested 2024-12-09T06:19:20,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160443 2024-12-09T06:19:20,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160443 2024-12-09T06:19:20,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160443) roll requested 2024-12-09T06:19:20,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160453 2024-12-09T06:19:20,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160453 2024-12-09T06:19:20,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160453) roll requested 2024-12-09T06:19:20,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160461 2024-12-09T06:19:20,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160461 2024-12-09T06:19:20,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160461) roll requested 2024-12-09T06:19:20,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160470 2024-12-09T06:19:20,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160470 2024-12-09T06:19:20,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160470) roll requested 2024-12-09T06:19:20,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160479 2024-12-09T06:19:20,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160479 2024-12-09T06:19:20,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160479) roll requested 2024-12-09T06:19:20,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160488 2024-12-09T06:19:20,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160488 2024-12-09T06:19:20,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160488) roll requested 2024-12-09T06:19:20,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160496 2024-12-09T06:19:20,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160496 2024-12-09T06:19:20,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160496) roll requested 2024-12-09T06:19:20,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160507 2024-12-09T06:19:20,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160507 2024-12-09T06:19:20,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160507) roll requested 2024-12-09T06:19:20,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160517 2024-12-09T06:19:20,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160517 2024-12-09T06:19:20,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160517) roll requested 2024-12-09T06:19:20,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160525 2024-12-09T06:19:20,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160525 2024-12-09T06:19:20,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160525) roll requested 2024-12-09T06:19:20,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160534 2024-12-09T06:19:20,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160534 2024-12-09T06:19:20,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160534) roll requested 2024-12-09T06:19:20,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160544 2024-12-09T06:19:20,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160544 2024-12-09T06:19:20,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160544) roll requested 2024-12-09T06:19:20,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160555 2024-12-09T06:19:20,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160555 2024-12-09T06:19:20,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160555) roll requested 2024-12-09T06:19:20,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160562 2024-12-09T06:19:20,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160562 2024-12-09T06:19:20,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160562) roll requested 2024-12-09T06:19:20,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160575 2024-12-09T06:19:20,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160575 2024-12-09T06:19:20,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160575) roll requested 2024-12-09T06:19:20,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160582 2024-12-09T06:19:20,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160582 2024-12-09T06:19:20,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160582) roll requested 2024-12-09T06:19:20,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160588 2024-12-09T06:19:20,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160588 2024-12-09T06:19:20,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160588) roll requested 2024-12-09T06:19:20,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160593 2024-12-09T06:19:20,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160593 2024-12-09T06:19:20,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160593) roll requested 2024-12-09T06:19:20,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160598 2024-12-09T06:19:20,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160598 2024-12-09T06:19:20,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160598) roll requested 2024-12-09T06:19:20,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160603 2024-12-09T06:19:20,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160603 2024-12-09T06:19:20,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160603) roll requested 2024-12-09T06:19:20,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160609 2024-12-09T06:19:20,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160609 2024-12-09T06:19:20,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160609) roll requested 2024-12-09T06:19:20,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160617 2024-12-09T06:19:20,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160617 2024-12-09T06:19:20,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160617) roll requested 2024-12-09T06:19:20,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160625 2024-12-09T06:19:20,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160625 2024-12-09T06:19:20,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160625) roll requested 2024-12-09T06:19:20,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160633 2024-12-09T06:19:20,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160633 2024-12-09T06:19:20,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160633) roll requested 2024-12-09T06:19:20,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160644 2024-12-09T06:19:20,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160644 2024-12-09T06:19:20,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160644) roll requested 2024-12-09T06:19:20,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160654 2024-12-09T06:19:20,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160654 2024-12-09T06:19:20,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160654) roll requested 2024-12-09T06:19:20,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160660 2024-12-09T06:19:20,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160660 2024-12-09T06:19:20,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160660) roll requested 2024-12-09T06:19:20,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160666 2024-12-09T06:19:20,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160666 2024-12-09T06:19:20,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160666) roll requested 2024-12-09T06:19:20,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160672 2024-12-09T06:19:20,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160672 2024-12-09T06:19:20,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160672) roll requested 2024-12-09T06:19:20,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160680 2024-12-09T06:19:20,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160680 2024-12-09T06:19:20,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160680) roll requested 2024-12-09T06:19:20,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160687 2024-12-09T06:19:20,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160687 2024-12-09T06:19:20,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160687) roll requested 2024-12-09T06:19:20,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160693 2024-12-09T06:19:20,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160693 2024-12-09T06:19:20,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160693) roll requested 2024-12-09T06:19:20,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160698 2024-12-09T06:19:20,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160698 2024-12-09T06:19:20,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160698) roll requested 2024-12-09T06:19:20,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160703 2024-12-09T06:19:20,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160703 2024-12-09T06:19:20,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160703) roll requested 2024-12-09T06:19:20,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160708 2024-12-09T06:19:20,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160708 2024-12-09T06:19:20,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160708) roll requested 2024-12-09T06:19:20,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160714 2024-12-09T06:19:20,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160714 2024-12-09T06:19:20,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160714) roll requested 2024-12-09T06:19:20,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160721 2024-12-09T06:19:20,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160721 2024-12-09T06:19:20,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160721) roll requested 2024-12-09T06:19:20,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160728 2024-12-09T06:19:20,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160728 2024-12-09T06:19:20,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160728) roll requested 2024-12-09T06:19:20,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160737 2024-12-09T06:19:20,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160737 2024-12-09T06:19:20,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160737) roll requested 2024-12-09T06:19:20,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160746 2024-12-09T06:19:20,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160746 2024-12-09T06:19:20,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160746) roll requested 2024-12-09T06:19:20,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160753 2024-12-09T06:19:20,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160753 2024-12-09T06:19:20,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160753) roll requested 2024-12-09T06:19:20,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160761 2024-12-09T06:19:20,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160761 2024-12-09T06:19:20,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160761) roll requested 2024-12-09T06:19:20,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160766 2024-12-09T06:19:20,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160766 2024-12-09T06:19:20,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160766) roll requested 2024-12-09T06:19:20,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160772 2024-12-09T06:19:20,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160772 2024-12-09T06:19:20,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160772) roll requested 2024-12-09T06:19:20,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160777 2024-12-09T06:19:20,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160777 2024-12-09T06:19:20,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160777) roll requested 2024-12-09T06:19:20,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160783 2024-12-09T06:19:20,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160783 2024-12-09T06:19:20,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160783) roll requested 2024-12-09T06:19:20,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160788 2024-12-09T06:19:20,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160788 2024-12-09T06:19:20,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160788) roll requested 2024-12-09T06:19:20,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160794 2024-12-09T06:19:20,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160794 2024-12-09T06:19:20,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160794) roll requested 2024-12-09T06:19:20,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160799 2024-12-09T06:19:20,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160799 2024-12-09T06:19:20,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160799) roll requested 2024-12-09T06:19:20,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160805 2024-12-09T06:19:20,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160805 2024-12-09T06:19:20,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160805) roll requested 2024-12-09T06:19:20,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160811 2024-12-09T06:19:20,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160811 2024-12-09T06:19:20,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160811) roll requested 2024-12-09T06:19:20,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160818 2024-12-09T06:19:20,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160818 2024-12-09T06:19:20,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160818) roll requested 2024-12-09T06:19:20,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160825 2024-12-09T06:19:20,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160825 2024-12-09T06:19:20,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160825) roll requested 2024-12-09T06:19:20,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160831 2024-12-09T06:19:20,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160831 2024-12-09T06:19:20,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160831) roll requested 2024-12-09T06:19:20,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160838 2024-12-09T06:19:20,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160838 2024-12-09T06:19:20,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160838) roll requested 2024-12-09T06:19:20,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160845 2024-12-09T06:19:20,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160845 2024-12-09T06:19:20,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160845) roll requested 2024-12-09T06:19:20,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160852 2024-12-09T06:19:20,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160852 2024-12-09T06:19:20,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160852) roll requested 2024-12-09T06:19:20,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160857 2024-12-09T06:19:20,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160857 2024-12-09T06:19:20,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160857) roll requested 2024-12-09T06:19:20,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160863 2024-12-09T06:19:20,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160863 2024-12-09T06:19:20,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160863) roll requested 2024-12-09T06:19:20,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160870 2024-12-09T06:19:20,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160870 2024-12-09T06:19:20,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160870) roll requested 2024-12-09T06:19:20,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160878 2024-12-09T06:19:20,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160878 2024-12-09T06:19:20,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160878) roll requested 2024-12-09T06:19:20,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160887 2024-12-09T06:19:20,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160887 2024-12-09T06:19:20,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160887) roll requested 2024-12-09T06:19:20,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160896 2024-12-09T06:19:20,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160896 2024-12-09T06:19:20,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160896) roll requested 2024-12-09T06:19:20,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160906 2024-12-09T06:19:20,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160906 2024-12-09T06:19:20,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160906) roll requested 2024-12-09T06:19:20,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160915 2024-12-09T06:19:20,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160915 2024-12-09T06:19:20,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160915) roll requested 2024-12-09T06:19:20,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160925 2024-12-09T06:19:20,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160925 2024-12-09T06:19:20,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160925) roll requested 2024-12-09T06:19:20,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160933 2024-12-09T06:19:20,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160933 2024-12-09T06:19:20,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160933) roll requested 2024-12-09T06:19:20,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160941 2024-12-09T06:19:20,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160941 2024-12-09T06:19:20,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160941) roll requested 2024-12-09T06:19:20,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160949 2024-12-09T06:19:20,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160949 2024-12-09T06:19:20,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160949) roll requested 2024-12-09T06:19:20,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160956 2024-12-09T06:19:20,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160956 2024-12-09T06:19:20,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160956) roll requested 2024-12-09T06:19:20,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160965 2024-12-09T06:19:20,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160965 2024-12-09T06:19:20,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160965) roll requested 2024-12-09T06:19:20,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160971 2024-12-09T06:19:20,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160971 2024-12-09T06:19:20,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160971) roll requested 2024-12-09T06:19:20,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160979 2024-12-09T06:19:20,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160979 2024-12-09T06:19:20,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160979) roll requested 2024-12-09T06:19:20,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160985 2024-12-09T06:19:20,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160985 2024-12-09T06:19:20,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160985) roll requested 2024-12-09T06:19:20,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160990 2024-12-09T06:19:20,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:20,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160990 2024-12-09T06:19:20,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:20,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:20,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:20,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:20,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160990) roll requested 2024-12-09T06:19:20,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725160996 2024-12-09T06:19:21,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160996 2024-12-09T06:19:21,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725160996) roll requested 2024-12-09T06:19:21,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161001 2024-12-09T06:19:21,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725160996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161001 2024-12-09T06:19:21,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161001) roll requested 2024-12-09T06:19:21,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161006 2024-12-09T06:19:21,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161006 2024-12-09T06:19:21,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161006) roll requested 2024-12-09T06:19:21,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161011 2024-12-09T06:19:21,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161011 2024-12-09T06:19:21,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161011) roll requested 2024-12-09T06:19:21,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161018 2024-12-09T06:19:21,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161018 2024-12-09T06:19:21,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161018) roll requested 2024-12-09T06:19:21,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161024 2024-12-09T06:19:21,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161024 2024-12-09T06:19:21,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161024) roll requested 2024-12-09T06:19:21,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161033 2024-12-09T06:19:21,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161033 2024-12-09T06:19:21,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161033) roll requested 2024-12-09T06:19:21,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161039 2024-12-09T06:19:21,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161039 2024-12-09T06:19:21,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161039) roll requested 2024-12-09T06:19:21,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161045 2024-12-09T06:19:21,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161045 2024-12-09T06:19:21,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161045) roll requested 2024-12-09T06:19:21,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161054 2024-12-09T06:19:21,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161054 2024-12-09T06:19:21,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161054) roll requested 2024-12-09T06:19:21,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161061 2024-12-09T06:19:21,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161061 2024-12-09T06:19:21,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161061) roll requested 2024-12-09T06:19:21,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161068 2024-12-09T06:19:21,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161068 2024-12-09T06:19:21,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161068) roll requested 2024-12-09T06:19:21,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161073 2024-12-09T06:19:21,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161073 2024-12-09T06:19:21,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161073) roll requested 2024-12-09T06:19:21,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161079 2024-12-09T06:19:21,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161079 2024-12-09T06:19:21,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161079) roll requested 2024-12-09T06:19:21,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161086 2024-12-09T06:19:21,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161086 2024-12-09T06:19:21,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161086) roll requested 2024-12-09T06:19:21,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161092 2024-12-09T06:19:21,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161092 2024-12-09T06:19:21,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161092) roll requested 2024-12-09T06:19:21,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161099 2024-12-09T06:19:21,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161099 2024-12-09T06:19:21,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161099) roll requested 2024-12-09T06:19:21,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161106 2024-12-09T06:19:21,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161106 2024-12-09T06:19:21,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161106) roll requested 2024-12-09T06:19:21,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161113 2024-12-09T06:19:21,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161113 2024-12-09T06:19:21,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161113) roll requested 2024-12-09T06:19:21,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161120 2024-12-09T06:19:21,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161120 2024-12-09T06:19:21,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161120) roll requested 2024-12-09T06:19:21,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161133 2024-12-09T06:19:21,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161133 2024-12-09T06:19:21,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161133) roll requested 2024-12-09T06:19:21,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161141 2024-12-09T06:19:21,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161141 2024-12-09T06:19:21,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161141) roll requested 2024-12-09T06:19:21,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161149 2024-12-09T06:19:21,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161149 2024-12-09T06:19:21,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161149) roll requested 2024-12-09T06:19:21,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161156 2024-12-09T06:19:21,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161156 2024-12-09T06:19:21,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161156) roll requested 2024-12-09T06:19:21,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161162 2024-12-09T06:19:21,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161162 2024-12-09T06:19:21,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161162) roll requested 2024-12-09T06:19:21,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161169 2024-12-09T06:19:21,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161169 2024-12-09T06:19:21,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161169) roll requested 2024-12-09T06:19:21,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161177 2024-12-09T06:19:21,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161177 2024-12-09T06:19:21,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161177) roll requested 2024-12-09T06:19:21,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161183 2024-12-09T06:19:21,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161183 2024-12-09T06:19:21,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161183) roll requested 2024-12-09T06:19:21,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161189 2024-12-09T06:19:21,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161189 2024-12-09T06:19:21,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161189) roll requested 2024-12-09T06:19:21,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161194 2024-12-09T06:19:21,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161194 2024-12-09T06:19:21,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161194) roll requested 2024-12-09T06:19:21,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161201 2024-12-09T06:19:21,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161201 2024-12-09T06:19:21,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161201) roll requested 2024-12-09T06:19:21,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161209 2024-12-09T06:19:21,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161209 2024-12-09T06:19:21,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161209) roll requested 2024-12-09T06:19:21,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161214 2024-12-09T06:19:21,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161214 2024-12-09T06:19:21,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161214) roll requested 2024-12-09T06:19:21,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161219 2024-12-09T06:19:21,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161219 2024-12-09T06:19:21,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161219) roll requested 2024-12-09T06:19:21,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161225 2024-12-09T06:19:21,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161225 2024-12-09T06:19:21,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161225) roll requested 2024-12-09T06:19:21,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161231 2024-12-09T06:19:21,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161231 2024-12-09T06:19:21,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161231) roll requested 2024-12-09T06:19:21,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161237 2024-12-09T06:19:21,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161237 2024-12-09T06:19:21,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161237) roll requested 2024-12-09T06:19:21,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161243 2024-12-09T06:19:21,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161243 2024-12-09T06:19:21,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161243) roll requested 2024-12-09T06:19:21,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161248 2024-12-09T06:19:21,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161248 2024-12-09T06:19:21,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161248) roll requested 2024-12-09T06:19:21,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161253 2024-12-09T06:19:21,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161253 2024-12-09T06:19:21,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161253) roll requested 2024-12-09T06:19:21,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161259 2024-12-09T06:19:21,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161259 2024-12-09T06:19:21,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161259) roll requested 2024-12-09T06:19:21,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161264 2024-12-09T06:19:21,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161264 2024-12-09T06:19:21,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161264) roll requested 2024-12-09T06:19:21,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161269 2024-12-09T06:19:21,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161269 2024-12-09T06:19:21,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161269) roll requested 2024-12-09T06:19:21,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161275 2024-12-09T06:19:21,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161275 2024-12-09T06:19:21,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161275) roll requested 2024-12-09T06:19:21,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161280 2024-12-09T06:19:21,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161280 2024-12-09T06:19:21,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161280) roll requested 2024-12-09T06:19:21,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161286 2024-12-09T06:19:21,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161286 2024-12-09T06:19:21,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161286) roll requested 2024-12-09T06:19:21,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161292 2024-12-09T06:19:21,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161292 2024-12-09T06:19:21,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161292) roll requested 2024-12-09T06:19:21,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161297 2024-12-09T06:19:21,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161297 2024-12-09T06:19:21,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161297) roll requested 2024-12-09T06:19:21,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161302 2024-12-09T06:19:21,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161302 2024-12-09T06:19:21,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161302) roll requested 2024-12-09T06:19:21,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161308 2024-12-09T06:19:21,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161308 2024-12-09T06:19:21,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161308) roll requested 2024-12-09T06:19:21,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161313 2024-12-09T06:19:21,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161313 2024-12-09T06:19:21,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161313) roll requested 2024-12-09T06:19:21,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161321 2024-12-09T06:19:21,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161321 2024-12-09T06:19:21,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161321) roll requested 2024-12-09T06:19:21,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161326 2024-12-09T06:19:21,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161326 2024-12-09T06:19:21,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161326) roll requested 2024-12-09T06:19:21,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161331 2024-12-09T06:19:21,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161331 2024-12-09T06:19:21,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161331) roll requested 2024-12-09T06:19:21,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161335 2024-12-09T06:19:21,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161335 2024-12-09T06:19:21,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161335) roll requested 2024-12-09T06:19:21,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161342 2024-12-09T06:19:21,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161342 2024-12-09T06:19:21,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161342) roll requested 2024-12-09T06:19:21,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161349 2024-12-09T06:19:21,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161349 2024-12-09T06:19:21,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161349) roll requested 2024-12-09T06:19:21,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161358 2024-12-09T06:19:21,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161358 2024-12-09T06:19:21,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161358) roll requested 2024-12-09T06:19:21,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161365 2024-12-09T06:19:21,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161365 2024-12-09T06:19:21,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161365) roll requested 2024-12-09T06:19:21,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161371 2024-12-09T06:19:21,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161371 2024-12-09T06:19:21,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161371) roll requested 2024-12-09T06:19:21,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161376 2024-12-09T06:19:21,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161376 2024-12-09T06:19:21,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161376) roll requested 2024-12-09T06:19:21,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161381 2024-12-09T06:19:21,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161381 2024-12-09T06:19:21,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161381) roll requested 2024-12-09T06:19:21,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161387 2024-12-09T06:19:21,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161387 2024-12-09T06:19:21,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161387) roll requested 2024-12-09T06:19:21,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161394 2024-12-09T06:19:21,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161394 2024-12-09T06:19:21,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161394) roll requested 2024-12-09T06:19:21,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161400 2024-12-09T06:19:21,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161400 2024-12-09T06:19:21,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161400) roll requested 2024-12-09T06:19:21,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161405 2024-12-09T06:19:21,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161405 2024-12-09T06:19:21,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161405) roll requested 2024-12-09T06:19:21,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161410 2024-12-09T06:19:21,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161410 2024-12-09T06:19:21,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161410) roll requested 2024-12-09T06:19:21,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161415 2024-12-09T06:19:21,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161415 2024-12-09T06:19:21,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161415) roll requested 2024-12-09T06:19:21,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161421 2024-12-09T06:19:21,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161421 2024-12-09T06:19:21,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161421) roll requested 2024-12-09T06:19:21,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161428 2024-12-09T06:19:21,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161428 2024-12-09T06:19:21,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161428) roll requested 2024-12-09T06:19:21,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161437 2024-12-09T06:19:21,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161437 2024-12-09T06:19:21,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161437) roll requested 2024-12-09T06:19:21,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161445 2024-12-09T06:19:21,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161445 2024-12-09T06:19:21,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161445) roll requested 2024-12-09T06:19:21,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161451 2024-12-09T06:19:21,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161451 2024-12-09T06:19:21,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161451) roll requested 2024-12-09T06:19:21,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161458 2024-12-09T06:19:21,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161458 2024-12-09T06:19:21,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161458) roll requested 2024-12-09T06:19:21,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161465 2024-12-09T06:19:21,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161465 2024-12-09T06:19:21,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161465) roll requested 2024-12-09T06:19:21,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161473 2024-12-09T06:19:21,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161473 2024-12-09T06:19:21,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161473) roll requested 2024-12-09T06:19:21,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161480 2024-12-09T06:19:21,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161480 2024-12-09T06:19:21,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161480) roll requested 2024-12-09T06:19:21,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161485 2024-12-09T06:19:21,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161485 2024-12-09T06:19:21,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161485) roll requested 2024-12-09T06:19:21,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161493 2024-12-09T06:19:21,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161493 2024-12-09T06:19:21,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161493) roll requested 2024-12-09T06:19:21,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161499 2024-12-09T06:19:21,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161499 2024-12-09T06:19:21,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161499) roll requested 2024-12-09T06:19:21,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161504 2024-12-09T06:19:21,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161504 2024-12-09T06:19:21,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161504) roll requested 2024-12-09T06:19:21,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161509 2024-12-09T06:19:21,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161509 2024-12-09T06:19:21,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161509) roll requested 2024-12-09T06:19:21,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161517 2024-12-09T06:19:21,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161517 2024-12-09T06:19:21,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161517) roll requested 2024-12-09T06:19:21,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161523 2024-12-09T06:19:21,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161523 2024-12-09T06:19:21,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161523) roll requested 2024-12-09T06:19:21,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161528 2024-12-09T06:19:21,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161528 2024-12-09T06:19:21,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161528) roll requested 2024-12-09T06:19:21,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161537 2024-12-09T06:19:21,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161537 2024-12-09T06:19:21,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161537) roll requested 2024-12-09T06:19:21,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161543 2024-12-09T06:19:21,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161543 2024-12-09T06:19:21,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161543) roll requested 2024-12-09T06:19:21,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161549 2024-12-09T06:19:21,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161549 2024-12-09T06:19:21,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161549) roll requested 2024-12-09T06:19:21,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161554 2024-12-09T06:19:21,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161554 2024-12-09T06:19:21,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161554) roll requested 2024-12-09T06:19:21,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161560 2024-12-09T06:19:21,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161560 2024-12-09T06:19:21,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161560) roll requested 2024-12-09T06:19:21,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161568 2024-12-09T06:19:21,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161568 2024-12-09T06:19:21,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161568) roll requested 2024-12-09T06:19:21,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161573 2024-12-09T06:19:21,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161573 2024-12-09T06:19:21,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161573) roll requested 2024-12-09T06:19:21,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161579 2024-12-09T06:19:21,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161579 2024-12-09T06:19:21,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161579) roll requested 2024-12-09T06:19:21,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161585 2024-12-09T06:19:21,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161585 2024-12-09T06:19:21,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161585) roll requested 2024-12-09T06:19:21,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161590 2024-12-09T06:19:21,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161590 2024-12-09T06:19:21,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161590) roll requested 2024-12-09T06:19:21,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161596 2024-12-09T06:19:21,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161596 2024-12-09T06:19:21,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161596) roll requested 2024-12-09T06:19:21,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161601 2024-12-09T06:19:21,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161601 2024-12-09T06:19:21,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161601) roll requested 2024-12-09T06:19:21,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161611 2024-12-09T06:19:21,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161611 2024-12-09T06:19:21,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161611) roll requested 2024-12-09T06:19:21,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161618 2024-12-09T06:19:21,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161618 2024-12-09T06:19:21,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161618) roll requested 2024-12-09T06:19:21,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161623 2024-12-09T06:19:21,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161623 2024-12-09T06:19:21,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161623) roll requested 2024-12-09T06:19:21,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161629 2024-12-09T06:19:21,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161629 2024-12-09T06:19:21,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161629) roll requested 2024-12-09T06:19:21,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161635 2024-12-09T06:19:21,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161635 2024-12-09T06:19:21,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161635) roll requested 2024-12-09T06:19:21,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161640 2024-12-09T06:19:21,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161640 2024-12-09T06:19:21,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161640) roll requested 2024-12-09T06:19:21,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161646 2024-12-09T06:19:21,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161646 2024-12-09T06:19:21,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161646) roll requested 2024-12-09T06:19:21,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161653 2024-12-09T06:19:21,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161653 2024-12-09T06:19:21,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161653) roll requested 2024-12-09T06:19:21,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161658 2024-12-09T06:19:21,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161658 2024-12-09T06:19:21,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161658) roll requested 2024-12-09T06:19:21,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161662 2024-12-09T06:19:21,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161662 2024-12-09T06:19:21,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161662) roll requested 2024-12-09T06:19:21,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161668 2024-12-09T06:19:21,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161668 2024-12-09T06:19:21,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161668) roll requested 2024-12-09T06:19:21,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161674 2024-12-09T06:19:21,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161674 2024-12-09T06:19:21,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161674) roll requested 2024-12-09T06:19:21,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161682 2024-12-09T06:19:21,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161682 2024-12-09T06:19:21,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161682) roll requested 2024-12-09T06:19:21,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161690 2024-12-09T06:19:21,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161690 2024-12-09T06:19:21,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161690) roll requested 2024-12-09T06:19:21,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161700 2024-12-09T06:19:21,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161700 2024-12-09T06:19:21,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161700) roll requested 2024-12-09T06:19:21,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161709 2024-12-09T06:19:21,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161709 2024-12-09T06:19:21,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161709) roll requested 2024-12-09T06:19:21,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161718 2024-12-09T06:19:21,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161718 2024-12-09T06:19:21,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161718) roll requested 2024-12-09T06:19:21,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161725 2024-12-09T06:19:21,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161725 2024-12-09T06:19:21,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161725) roll requested 2024-12-09T06:19:21,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161733 2024-12-09T06:19:21,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161733 2024-12-09T06:19:21,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161733) roll requested 2024-12-09T06:19:21,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161741 2024-12-09T06:19:21,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161741 2024-12-09T06:19:21,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161741) roll requested 2024-12-09T06:19:21,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161746 2024-12-09T06:19:21,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161746 2024-12-09T06:19:21,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161746) roll requested 2024-12-09T06:19:21,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161752 2024-12-09T06:19:21,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161752 2024-12-09T06:19:21,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161752) roll requested 2024-12-09T06:19:21,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161761 2024-12-09T06:19:21,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161761 2024-12-09T06:19:21,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161761) roll requested 2024-12-09T06:19:21,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161771 2024-12-09T06:19:21,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161771 2024-12-09T06:19:21,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161771) roll requested 2024-12-09T06:19:21,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161776 2024-12-09T06:19:21,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161776 2024-12-09T06:19:21,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161776) roll requested 2024-12-09T06:19:21,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161783 2024-12-09T06:19:21,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161783 2024-12-09T06:19:21,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161783) roll requested 2024-12-09T06:19:21,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161788 2024-12-09T06:19:21,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161788 2024-12-09T06:19:21,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161788) roll requested 2024-12-09T06:19:21,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161793 2024-12-09T06:19:21,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161793 2024-12-09T06:19:21,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161793) roll requested 2024-12-09T06:19:21,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161798 2024-12-09T06:19:21,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161798 2024-12-09T06:19:21,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161798) roll requested 2024-12-09T06:19:21,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161803 2024-12-09T06:19:21,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161803 2024-12-09T06:19:21,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161803) roll requested 2024-12-09T06:19:21,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161809 2024-12-09T06:19:21,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161809 2024-12-09T06:19:21,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161809) roll requested 2024-12-09T06:19:21,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161815 2024-12-09T06:19:21,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161815 2024-12-09T06:19:21,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161815) roll requested 2024-12-09T06:19:21,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161822 2024-12-09T06:19:21,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161822 2024-12-09T06:19:21,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161822) roll requested 2024-12-09T06:19:21,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161829 2024-12-09T06:19:21,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161829 2024-12-09T06:19:21,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161829) roll requested 2024-12-09T06:19:21,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161838 2024-12-09T06:19:21,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161838 2024-12-09T06:19:21,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161838) roll requested 2024-12-09T06:19:21,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161845 2024-12-09T06:19:21,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161845 2024-12-09T06:19:21,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161845) roll requested 2024-12-09T06:19:21,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161850 2024-12-09T06:19:21,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161850 2024-12-09T06:19:21,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161850) roll requested 2024-12-09T06:19:21,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161856 2024-12-09T06:19:21,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161856 2024-12-09T06:19:21,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161856) roll requested 2024-12-09T06:19:21,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161861 2024-12-09T06:19:21,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161861 2024-12-09T06:19:21,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161861) roll requested 2024-12-09T06:19:21,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161867 2024-12-09T06:19:21,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161867 2024-12-09T06:19:21,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161867) roll requested 2024-12-09T06:19:21,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161874 2024-12-09T06:19:21,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161874 2024-12-09T06:19:21,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161874) roll requested 2024-12-09T06:19:21,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161880 2024-12-09T06:19:21,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161880 2024-12-09T06:19:21,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161880) roll requested 2024-12-09T06:19:21,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161886 2024-12-09T06:19:21,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161886 2024-12-09T06:19:21,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161886) roll requested 2024-12-09T06:19:21,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161893 2024-12-09T06:19:21,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161893 2024-12-09T06:19:21,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161893) roll requested 2024-12-09T06:19:21,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161900 2024-12-09T06:19:21,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161900 2024-12-09T06:19:21,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161900) roll requested 2024-12-09T06:19:21,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161906 2024-12-09T06:19:21,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161906 2024-12-09T06:19:21,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161906) roll requested 2024-12-09T06:19:21,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161911 2024-12-09T06:19:21,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161911 2024-12-09T06:19:21,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161911) roll requested 2024-12-09T06:19:21,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161919 2024-12-09T06:19:21,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161919 2024-12-09T06:19:21,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161919) roll requested 2024-12-09T06:19:21,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161926 2024-12-09T06:19:21,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161926 2024-12-09T06:19:21,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161926) roll requested 2024-12-09T06:19:21,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161934 2024-12-09T06:19:21,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161934 2024-12-09T06:19:21,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161934) roll requested 2024-12-09T06:19:21,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161944 2024-12-09T06:19:21,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161944 2024-12-09T06:19:21,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161944) roll requested 2024-12-09T06:19:21,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161952 2024-12-09T06:19:21,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161952 2024-12-09T06:19:21,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161952) roll requested 2024-12-09T06:19:21,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161958 2024-12-09T06:19:21,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161958 2024-12-09T06:19:21,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161958) roll requested 2024-12-09T06:19:21,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161965 2024-12-09T06:19:21,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161965 2024-12-09T06:19:21,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161965) roll requested 2024-12-09T06:19:21,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161973 2024-12-09T06:19:21,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161973 2024-12-09T06:19:21,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161973) roll requested 2024-12-09T06:19:21,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161980 2024-12-09T06:19:21,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161980 2024-12-09T06:19:21,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161980) roll requested 2024-12-09T06:19:21,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161987 2024-12-09T06:19:21,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:21,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161987 2024-12-09T06:19:21,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:21,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:21,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:21,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:21,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161987) roll requested 2024-12-09T06:19:21,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725161997 2024-12-09T06:19:22,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161997 2024-12-09T06:19:22,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725161997) roll requested 2024-12-09T06:19:22,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162005 2024-12-09T06:19:22,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725161997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162005 2024-12-09T06:19:22,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162005) roll requested 2024-12-09T06:19:22,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162013 2024-12-09T06:19:22,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162013 2024-12-09T06:19:22,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162013) roll requested 2024-12-09T06:19:22,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162022 2024-12-09T06:19:22,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162022 2024-12-09T06:19:22,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162022) roll requested 2024-12-09T06:19:22,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162030 2024-12-09T06:19:22,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162030 2024-12-09T06:19:22,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162030) roll requested 2024-12-09T06:19:22,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162038 2024-12-09T06:19:22,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162038 2024-12-09T06:19:22,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162038) roll requested 2024-12-09T06:19:22,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162046 2024-12-09T06:19:22,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162046 2024-12-09T06:19:22,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162046) roll requested 2024-12-09T06:19:22,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162053 2024-12-09T06:19:22,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162053 2024-12-09T06:19:22,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162053) roll requested 2024-12-09T06:19:22,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162060 2024-12-09T06:19:22,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162060 2024-12-09T06:19:22,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162060) roll requested 2024-12-09T06:19:22,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162069 2024-12-09T06:19:22,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162069 2024-12-09T06:19:22,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162069) roll requested 2024-12-09T06:19:22,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162074 2024-12-09T06:19:22,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162074 2024-12-09T06:19:22,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162074) roll requested 2024-12-09T06:19:22,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162081 2024-12-09T06:19:22,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162081 2024-12-09T06:19:22,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162081) roll requested 2024-12-09T06:19:22,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162087 2024-12-09T06:19:22,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162087 2024-12-09T06:19:22,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162087) roll requested 2024-12-09T06:19:22,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162095 2024-12-09T06:19:22,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162095 2024-12-09T06:19:22,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162095) roll requested 2024-12-09T06:19:22,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162104 2024-12-09T06:19:22,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162104 2024-12-09T06:19:22,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162104) roll requested 2024-12-09T06:19:22,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162111 2024-12-09T06:19:22,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162111 2024-12-09T06:19:22,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162111) roll requested 2024-12-09T06:19:22,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162117 2024-12-09T06:19:22,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162117 2024-12-09T06:19:22,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162117) roll requested 2024-12-09T06:19:22,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162124 2024-12-09T06:19:22,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162124 2024-12-09T06:19:22,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162124) roll requested 2024-12-09T06:19:22,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162130 2024-12-09T06:19:22,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162130 2024-12-09T06:19:22,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162130) roll requested 2024-12-09T06:19:22,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162136 2024-12-09T06:19:22,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162136 2024-12-09T06:19:22,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162136) roll requested 2024-12-09T06:19:22,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162142 2024-12-09T06:19:22,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162142 2024-12-09T06:19:22,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162142) roll requested 2024-12-09T06:19:22,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162148 2024-12-09T06:19:22,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162148 2024-12-09T06:19:22,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162148) roll requested 2024-12-09T06:19:22,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162154 2024-12-09T06:19:22,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162154 2024-12-09T06:19:22,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162154) roll requested 2024-12-09T06:19:22,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162160 2024-12-09T06:19:22,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162160 2024-12-09T06:19:22,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162160) roll requested 2024-12-09T06:19:22,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162167 2024-12-09T06:19:22,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162167 2024-12-09T06:19:22,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162167) roll requested 2024-12-09T06:19:22,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162173 2024-12-09T06:19:22,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162173 2024-12-09T06:19:22,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162173) roll requested 2024-12-09T06:19:22,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162179 2024-12-09T06:19:22,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162179 2024-12-09T06:19:22,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162179) roll requested 2024-12-09T06:19:22,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162184 2024-12-09T06:19:22,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162184 2024-12-09T06:19:22,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162184) roll requested 2024-12-09T06:19:22,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162190 2024-12-09T06:19:22,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162190 2024-12-09T06:19:22,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162190) roll requested 2024-12-09T06:19:22,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162195 2024-12-09T06:19:22,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162195 2024-12-09T06:19:22,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162195) roll requested 2024-12-09T06:19:22,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162201 2024-12-09T06:19:22,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162201 2024-12-09T06:19:22,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162201) roll requested 2024-12-09T06:19:22,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162205 2024-12-09T06:19:22,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162205 2024-12-09T06:19:22,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162205) roll requested 2024-12-09T06:19:22,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162211 2024-12-09T06:19:22,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162211 2024-12-09T06:19:22,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162211) roll requested 2024-12-09T06:19:22,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162218 2024-12-09T06:19:22,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162218 2024-12-09T06:19:22,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162218) roll requested 2024-12-09T06:19:22,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162225 2024-12-09T06:19:22,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162225 2024-12-09T06:19:22,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162225) roll requested 2024-12-09T06:19:22,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162234 2024-12-09T06:19:22,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162234 2024-12-09T06:19:22,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162234) roll requested 2024-12-09T06:19:22,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162241 2024-12-09T06:19:22,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162241 2024-12-09T06:19:22,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162241) roll requested 2024-12-09T06:19:22,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162249 2024-12-09T06:19:22,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162249 2024-12-09T06:19:22,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162249) roll requested 2024-12-09T06:19:22,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162257 2024-12-09T06:19:22,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162257 2024-12-09T06:19:22,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162257) roll requested 2024-12-09T06:19:22,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162265 2024-12-09T06:19:22,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162265 2024-12-09T06:19:22,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162265) roll requested 2024-12-09T06:19:22,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162400 2024-12-09T06:19:22,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162400 2024-12-09T06:19:22,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162400) roll requested 2024-12-09T06:19:22,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162405 2024-12-09T06:19:22,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162405 2024-12-09T06:19:22,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162405) roll requested 2024-12-09T06:19:22,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162412 2024-12-09T06:19:22,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162412 2024-12-09T06:19:22,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162412) roll requested 2024-12-09T06:19:22,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162418 2024-12-09T06:19:22,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162418 2024-12-09T06:19:22,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162418) roll requested 2024-12-09T06:19:22,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162423 2024-12-09T06:19:22,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162423 2024-12-09T06:19:22,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162423) roll requested 2024-12-09T06:19:22,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162429 2024-12-09T06:19:22,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162429 2024-12-09T06:19:22,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162429) roll requested 2024-12-09T06:19:22,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162434 2024-12-09T06:19:22,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162434 2024-12-09T06:19:22,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162434) roll requested 2024-12-09T06:19:22,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162440 2024-12-09T06:19:22,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162440 2024-12-09T06:19:22,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162440) roll requested 2024-12-09T06:19:22,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162446 2024-12-09T06:19:22,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162446 2024-12-09T06:19:22,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162446) roll requested 2024-12-09T06:19:22,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162457 2024-12-09T06:19:22,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162457 2024-12-09T06:19:22,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162457) roll requested 2024-12-09T06:19:22,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162464 2024-12-09T06:19:22,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162464 2024-12-09T06:19:22,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162464) roll requested 2024-12-09T06:19:22,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162470 2024-12-09T06:19:22,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162470 2024-12-09T06:19:22,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162470) roll requested 2024-12-09T06:19:22,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162475 2024-12-09T06:19:22,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162475 2024-12-09T06:19:22,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162475) roll requested 2024-12-09T06:19:22,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162481 2024-12-09T06:19:22,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162481 2024-12-09T06:19:22,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162481) roll requested 2024-12-09T06:19:22,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162488 2024-12-09T06:19:22,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162488 2024-12-09T06:19:22,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162488) roll requested 2024-12-09T06:19:22,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162493 2024-12-09T06:19:22,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162493 2024-12-09T06:19:22,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162493) roll requested 2024-12-09T06:19:22,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162498 2024-12-09T06:19:22,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162498 2024-12-09T06:19:22,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162498) roll requested 2024-12-09T06:19:22,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162504 2024-12-09T06:19:22,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162504 2024-12-09T06:19:22,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162504) roll requested 2024-12-09T06:19:22,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162511 2024-12-09T06:19:22,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162511 2024-12-09T06:19:22,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162511) roll requested 2024-12-09T06:19:22,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162519 2024-12-09T06:19:22,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162519 2024-12-09T06:19:22,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162519) roll requested 2024-12-09T06:19:22,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162526 2024-12-09T06:19:22,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162526 2024-12-09T06:19:22,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162526) roll requested 2024-12-09T06:19:22,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162536 2024-12-09T06:19:22,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162536 2024-12-09T06:19:22,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162536) roll requested 2024-12-09T06:19:22,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162544 2024-12-09T06:19:22,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162544 2024-12-09T06:19:22,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162544) roll requested 2024-12-09T06:19:22,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162551 2024-12-09T06:19:22,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162551 2024-12-09T06:19:22,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162551) roll requested 2024-12-09T06:19:22,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162558 2024-12-09T06:19:22,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162558 2024-12-09T06:19:22,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162558) roll requested 2024-12-09T06:19:22,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162565 2024-12-09T06:19:22,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162565 2024-12-09T06:19:22,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162565) roll requested 2024-12-09T06:19:22,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162572 2024-12-09T06:19:22,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162572 2024-12-09T06:19:22,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162572) roll requested 2024-12-09T06:19:22,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162579 2024-12-09T06:19:22,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162579 2024-12-09T06:19:22,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162579) roll requested 2024-12-09T06:19:22,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162588 2024-12-09T06:19:22,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162588 2024-12-09T06:19:22,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162588) roll requested 2024-12-09T06:19:22,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162597 2024-12-09T06:19:22,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162597 2024-12-09T06:19:22,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162597) roll requested 2024-12-09T06:19:22,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162605 2024-12-09T06:19:22,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162605 2024-12-09T06:19:22,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162605) roll requested 2024-12-09T06:19:22,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162610 2024-12-09T06:19:22,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162610 2024-12-09T06:19:22,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162610) roll requested 2024-12-09T06:19:22,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162616 2024-12-09T06:19:22,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162616 2024-12-09T06:19:22,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162616) roll requested 2024-12-09T06:19:22,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162620 2024-12-09T06:19:22,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162620 2024-12-09T06:19:22,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162620) roll requested 2024-12-09T06:19:22,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162626 2024-12-09T06:19:22,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162626 2024-12-09T06:19:22,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162626) roll requested 2024-12-09T06:19:22,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162634 2024-12-09T06:19:22,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162634 2024-12-09T06:19:22,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162634) roll requested 2024-12-09T06:19:22,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162641 2024-12-09T06:19:22,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162641 2024-12-09T06:19:22,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162641) roll requested 2024-12-09T06:19:22,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162650 2024-12-09T06:19:22,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162650 2024-12-09T06:19:22,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162650) roll requested 2024-12-09T06:19:22,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162659 2024-12-09T06:19:22,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162659 2024-12-09T06:19:22,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162659) roll requested 2024-12-09T06:19:22,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162668 2024-12-09T06:19:22,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162668 2024-12-09T06:19:22,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162668) roll requested 2024-12-09T06:19:22,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162676 2024-12-09T06:19:22,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162676 2024-12-09T06:19:22,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162676) roll requested 2024-12-09T06:19:22,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162685 2024-12-09T06:19:22,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162685 2024-12-09T06:19:22,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162685) roll requested 2024-12-09T06:19:22,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162694 2024-12-09T06:19:22,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162694 2024-12-09T06:19:22,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162694) roll requested 2024-12-09T06:19:22,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162704 2024-12-09T06:19:22,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162704 2024-12-09T06:19:22,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162704) roll requested 2024-12-09T06:19:22,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162712 2024-12-09T06:19:22,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162712 2024-12-09T06:19:22,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162712) roll requested 2024-12-09T06:19:22,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162721 2024-12-09T06:19:22,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162721 2024-12-09T06:19:22,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162721) roll requested 2024-12-09T06:19:22,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162729 2024-12-09T06:19:22,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162729 2024-12-09T06:19:22,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162729) roll requested 2024-12-09T06:19:22,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162736 2024-12-09T06:19:22,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162736 2024-12-09T06:19:22,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162736) roll requested 2024-12-09T06:19:22,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162744 2024-12-09T06:19:22,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162744 2024-12-09T06:19:22,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162744) roll requested 2024-12-09T06:19:22,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162752 2024-12-09T06:19:22,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162752 2024-12-09T06:19:22,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162752) roll requested 2024-12-09T06:19:22,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162760 2024-12-09T06:19:22,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162760 2024-12-09T06:19:22,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162760) roll requested 2024-12-09T06:19:22,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162767 2024-12-09T06:19:22,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162767 2024-12-09T06:19:22,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162767) roll requested 2024-12-09T06:19:22,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162774 2024-12-09T06:19:22,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162774 2024-12-09T06:19:22,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162774) roll requested 2024-12-09T06:19:22,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162782 2024-12-09T06:19:22,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162782 2024-12-09T06:19:22,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162782) roll requested 2024-12-09T06:19:22,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162793 2024-12-09T06:19:22,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162793 2024-12-09T06:19:22,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162793) roll requested 2024-12-09T06:19:22,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162800 2024-12-09T06:19:22,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162800 2024-12-09T06:19:22,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162800) roll requested 2024-12-09T06:19:22,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162805 2024-12-09T06:19:22,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162805 2024-12-09T06:19:22,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162805) roll requested 2024-12-09T06:19:22,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162812 2024-12-09T06:19:22,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162812 2024-12-09T06:19:22,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162812) roll requested 2024-12-09T06:19:22,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162820 2024-12-09T06:19:22,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162820 2024-12-09T06:19:22,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162820) roll requested 2024-12-09T06:19:22,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162825 2024-12-09T06:19:22,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162825 2024-12-09T06:19:22,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162825) roll requested 2024-12-09T06:19:22,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162829 2024-12-09T06:19:22,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162829 2024-12-09T06:19:22,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162829) roll requested 2024-12-09T06:19:22,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162835 2024-12-09T06:19:22,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162835 2024-12-09T06:19:22,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162835) roll requested 2024-12-09T06:19:22,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162842 2024-12-09T06:19:22,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162842 2024-12-09T06:19:22,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162842) roll requested 2024-12-09T06:19:22,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162848 2024-12-09T06:19:22,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162848 2024-12-09T06:19:22,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162848) roll requested 2024-12-09T06:19:22,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162854 2024-12-09T06:19:22,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162854 2024-12-09T06:19:22,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162854) roll requested 2024-12-09T06:19:22,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162862 2024-12-09T06:19:22,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162862 2024-12-09T06:19:22,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162862) roll requested 2024-12-09T06:19:22,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162871 2024-12-09T06:19:22,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162871 2024-12-09T06:19:22,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162871) roll requested 2024-12-09T06:19:22,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162880 2024-12-09T06:19:22,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162880 2024-12-09T06:19:22,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162880) roll requested 2024-12-09T06:19:22,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162890 2024-12-09T06:19:22,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162890 2024-12-09T06:19:22,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162890) roll requested 2024-12-09T06:19:22,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162897 2024-12-09T06:19:22,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162897 2024-12-09T06:19:22,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162897) roll requested 2024-12-09T06:19:22,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162904 2024-12-09T06:19:22,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162904 2024-12-09T06:19:22,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162904) roll requested 2024-12-09T06:19:22,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162913 2024-12-09T06:19:22,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162913 2024-12-09T06:19:22,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162913) roll requested 2024-12-09T06:19:22,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162923 2024-12-09T06:19:22,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162923 2024-12-09T06:19:22,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162923) roll requested 2024-12-09T06:19:22,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162931 2024-12-09T06:19:22,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162931 2024-12-09T06:19:22,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162931) roll requested 2024-12-09T06:19:22,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162939 2024-12-09T06:19:22,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162939 2024-12-09T06:19:22,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162939) roll requested 2024-12-09T06:19:22,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162948 2024-12-09T06:19:22,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162948 2024-12-09T06:19:22,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162948) roll requested 2024-12-09T06:19:22,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162957 2024-12-09T06:19:22,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162957 2024-12-09T06:19:22,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162957) roll requested 2024-12-09T06:19:22,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162966 2024-12-09T06:19:22,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162966 2024-12-09T06:19:22,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162966) roll requested 2024-12-09T06:19:22,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162973 2024-12-09T06:19:22,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162973 2024-12-09T06:19:22,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162973) roll requested 2024-12-09T06:19:22,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162983 2024-12-09T06:19:22,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162983 2024-12-09T06:19:22,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:22,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:22,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:22,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:22,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162983) roll requested 2024-12-09T06:19:22,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725162991 2024-12-09T06:19:22,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:22,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162991 2024-12-09T06:19:23,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725162991) roll requested 2024-12-09T06:19:23,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163000 2024-12-09T06:19:23,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725162991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163000 2024-12-09T06:19:23,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163000) roll requested 2024-12-09T06:19:23,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163005 2024-12-09T06:19:23,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163005 2024-12-09T06:19:23,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163005) roll requested 2024-12-09T06:19:23,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163011 2024-12-09T06:19:23,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163011 2024-12-09T06:19:23,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163011) roll requested 2024-12-09T06:19:23,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163018 2024-12-09T06:19:23,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163018 2024-12-09T06:19:23,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163018) roll requested 2024-12-09T06:19:23,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163025 2024-12-09T06:19:23,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163025 2024-12-09T06:19:23,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163025) roll requested 2024-12-09T06:19:23,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163033 2024-12-09T06:19:23,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163033 2024-12-09T06:19:23,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163033) roll requested 2024-12-09T06:19:23,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163041 2024-12-09T06:19:23,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163041 2024-12-09T06:19:23,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163041) roll requested 2024-12-09T06:19:23,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163047 2024-12-09T06:19:23,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163047 2024-12-09T06:19:23,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163047) roll requested 2024-12-09T06:19:23,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163054 2024-12-09T06:19:23,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163054 2024-12-09T06:19:23,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163054) roll requested 2024-12-09T06:19:23,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163063 2024-12-09T06:19:23,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163063 2024-12-09T06:19:23,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163063) roll requested 2024-12-09T06:19:23,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163072 2024-12-09T06:19:23,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163072 2024-12-09T06:19:23,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163072) roll requested 2024-12-09T06:19:23,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163083 2024-12-09T06:19:23,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163083 2024-12-09T06:19:23,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163083) roll requested 2024-12-09T06:19:23,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163090 2024-12-09T06:19:23,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163090 2024-12-09T06:19:23,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163090) roll requested 2024-12-09T06:19:23,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163097 2024-12-09T06:19:23,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163097 2024-12-09T06:19:23,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163097) roll requested 2024-12-09T06:19:23,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163104 2024-12-09T06:19:23,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163104 2024-12-09T06:19:23,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163104) roll requested 2024-12-09T06:19:23,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163109 2024-12-09T06:19:23,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163109 2024-12-09T06:19:23,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163109) roll requested 2024-12-09T06:19:23,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163116 2024-12-09T06:19:23,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163116 2024-12-09T06:19:23,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163116) roll requested 2024-12-09T06:19:23,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163122 2024-12-09T06:19:23,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163122 2024-12-09T06:19:23,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163122) roll requested 2024-12-09T06:19:23,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163129 2024-12-09T06:19:23,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163129 2024-12-09T06:19:23,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163129) roll requested 2024-12-09T06:19:23,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163136 2024-12-09T06:19:23,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163136 2024-12-09T06:19:23,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163136) roll requested 2024-12-09T06:19:23,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163144 2024-12-09T06:19:23,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163144 2024-12-09T06:19:23,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163144) roll requested 2024-12-09T06:19:23,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163152 2024-12-09T06:19:23,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163152 2024-12-09T06:19:23,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163152) roll requested 2024-12-09T06:19:23,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163161 2024-12-09T06:19:23,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163161 2024-12-09T06:19:23,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163161) roll requested 2024-12-09T06:19:23,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163170 2024-12-09T06:19:23,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163170 2024-12-09T06:19:23,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163170) roll requested 2024-12-09T06:19:23,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163179 2024-12-09T06:19:23,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163179 2024-12-09T06:19:23,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163179) roll requested 2024-12-09T06:19:23,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163189 2024-12-09T06:19:23,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163189 2024-12-09T06:19:23,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163189) roll requested 2024-12-09T06:19:23,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163197 2024-12-09T06:19:23,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163197 2024-12-09T06:19:23,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163197) roll requested 2024-12-09T06:19:23,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163203 2024-12-09T06:19:23,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163203 2024-12-09T06:19:23,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163203) roll requested 2024-12-09T06:19:23,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163212 2024-12-09T06:19:23,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163212 2024-12-09T06:19:23,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163212) roll requested 2024-12-09T06:19:23,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163221 2024-12-09T06:19:23,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163221 2024-12-09T06:19:23,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163221) roll requested 2024-12-09T06:19:23,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163229 2024-12-09T06:19:23,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163229 2024-12-09T06:19:23,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163229) roll requested 2024-12-09T06:19:23,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163237 2024-12-09T06:19:23,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163237 2024-12-09T06:19:23,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163237) roll requested 2024-12-09T06:19:23,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163245 2024-12-09T06:19:23,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163245 2024-12-09T06:19:23,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163245) roll requested 2024-12-09T06:19:23,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163254 2024-12-09T06:19:23,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163254 2024-12-09T06:19:23,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163254) roll requested 2024-12-09T06:19:23,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163261 2024-12-09T06:19:23,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163261 2024-12-09T06:19:23,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163261) roll requested 2024-12-09T06:19:23,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163269 2024-12-09T06:19:23,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163269 2024-12-09T06:19:23,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163269) roll requested 2024-12-09T06:19:23,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163278 2024-12-09T06:19:23,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163278 2024-12-09T06:19:23,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163278) roll requested 2024-12-09T06:19:23,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163285 2024-12-09T06:19:23,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163285 2024-12-09T06:19:23,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163285) roll requested 2024-12-09T06:19:23,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163292 2024-12-09T06:19:23,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163292 2024-12-09T06:19:23,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163292) roll requested 2024-12-09T06:19:23,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163300 2024-12-09T06:19:23,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163300 2024-12-09T06:19:23,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163300) roll requested 2024-12-09T06:19:23,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163309 2024-12-09T06:19:23,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163309 2024-12-09T06:19:23,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163309) roll requested 2024-12-09T06:19:23,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163317 2024-12-09T06:19:23,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163317 2024-12-09T06:19:23,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163317) roll requested 2024-12-09T06:19:23,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163325 2024-12-09T06:19:23,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163325 2024-12-09T06:19:23,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163325) roll requested 2024-12-09T06:19:23,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163332 2024-12-09T06:19:23,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163332 2024-12-09T06:19:23,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163332) roll requested 2024-12-09T06:19:23,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163339 2024-12-09T06:19:23,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163339 2024-12-09T06:19:23,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163339) roll requested 2024-12-09T06:19:23,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163347 2024-12-09T06:19:23,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163347 2024-12-09T06:19:23,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163347) roll requested 2024-12-09T06:19:23,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163355 2024-12-09T06:19:23,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163355 2024-12-09T06:19:23,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163355) roll requested 2024-12-09T06:19:23,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163366 2024-12-09T06:19:23,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163366 2024-12-09T06:19:23,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163366) roll requested 2024-12-09T06:19:23,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163374 2024-12-09T06:19:23,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163374 2024-12-09T06:19:23,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163374) roll requested 2024-12-09T06:19:23,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163380 2024-12-09T06:19:23,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163380 2024-12-09T06:19:23,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163380) roll requested 2024-12-09T06:19:23,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163387 2024-12-09T06:19:23,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163387 2024-12-09T06:19:23,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163387) roll requested 2024-12-09T06:19:23,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163395 2024-12-09T06:19:23,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163395 2024-12-09T06:19:23,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163395) roll requested 2024-12-09T06:19:23,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163404 2024-12-09T06:19:23,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163404 2024-12-09T06:19:23,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163404) roll requested 2024-12-09T06:19:23,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163413 2024-12-09T06:19:23,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163413 2024-12-09T06:19:23,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163413) roll requested 2024-12-09T06:19:23,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163423 2024-12-09T06:19:23,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163423 2024-12-09T06:19:23,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163423) roll requested 2024-12-09T06:19:23,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163428 2024-12-09T06:19:23,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163428 2024-12-09T06:19:23,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163428) roll requested 2024-12-09T06:19:23,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163433 2024-12-09T06:19:23,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163433 2024-12-09T06:19:23,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163433) roll requested 2024-12-09T06:19:23,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163440 2024-12-09T06:19:23,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163440 2024-12-09T06:19:23,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163440) roll requested 2024-12-09T06:19:23,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163448 2024-12-09T06:19:23,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163448 2024-12-09T06:19:23,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163448) roll requested 2024-12-09T06:19:23,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163456 2024-12-09T06:19:23,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163456 2024-12-09T06:19:23,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163456) roll requested 2024-12-09T06:19:23,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163466 2024-12-09T06:19:23,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163466 2024-12-09T06:19:23,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163466) roll requested 2024-12-09T06:19:23,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163475 2024-12-09T06:19:23,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163475 2024-12-09T06:19:23,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163475) roll requested 2024-12-09T06:19:23,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163483 2024-12-09T06:19:23,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163483 2024-12-09T06:19:23,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163483) roll requested 2024-12-09T06:19:23,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163492 2024-12-09T06:19:23,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163492 2024-12-09T06:19:23,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163492) roll requested 2024-12-09T06:19:23,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163499 2024-12-09T06:19:23,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163499 2024-12-09T06:19:23,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163499) roll requested 2024-12-09T06:19:23,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163504 2024-12-09T06:19:23,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163504 2024-12-09T06:19:23,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163504) roll requested 2024-12-09T06:19:23,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163509 2024-12-09T06:19:23,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163509 2024-12-09T06:19:23,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163509) roll requested 2024-12-09T06:19:23,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163514 2024-12-09T06:19:23,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163514 2024-12-09T06:19:23,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163514) roll requested 2024-12-09T06:19:23,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163521 2024-12-09T06:19:23,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163521 2024-12-09T06:19:23,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163521) roll requested 2024-12-09T06:19:23,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163531 2024-12-09T06:19:23,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163531 2024-12-09T06:19:23,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163531) roll requested 2024-12-09T06:19:23,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163538 2024-12-09T06:19:23,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163538 2024-12-09T06:19:23,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163538) roll requested 2024-12-09T06:19:23,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163545 2024-12-09T06:19:23,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163545 2024-12-09T06:19:23,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163545) roll requested 2024-12-09T06:19:23,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163552 2024-12-09T06:19:23,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163552 2024-12-09T06:19:23,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163552) roll requested 2024-12-09T06:19:23,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163561 2024-12-09T06:19:23,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163561 2024-12-09T06:19:23,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163561) roll requested 2024-12-09T06:19:23,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163569 2024-12-09T06:19:23,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163569 2024-12-09T06:19:23,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163569) roll requested 2024-12-09T06:19:23,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163577 2024-12-09T06:19:23,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163577 2024-12-09T06:19:23,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163577) roll requested 2024-12-09T06:19:23,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163585 2024-12-09T06:19:23,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163585 2024-12-09T06:19:23,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163585) roll requested 2024-12-09T06:19:23,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163594 2024-12-09T06:19:23,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163594 2024-12-09T06:19:23,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163594) roll requested 2024-12-09T06:19:23,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163603 2024-12-09T06:19:23,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163603 2024-12-09T06:19:23,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163603) roll requested 2024-12-09T06:19:23,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163611 2024-12-09T06:19:23,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163611 2024-12-09T06:19:23,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163611) roll requested 2024-12-09T06:19:23,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163623 2024-12-09T06:19:23,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163623 2024-12-09T06:19:23,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163623) roll requested 2024-12-09T06:19:23,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163631 2024-12-09T06:19:23,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163631 2024-12-09T06:19:23,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163631) roll requested 2024-12-09T06:19:23,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163638 2024-12-09T06:19:23,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163638 2024-12-09T06:19:23,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163638) roll requested 2024-12-09T06:19:23,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163644 2024-12-09T06:19:23,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163644 2024-12-09T06:19:23,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163644) roll requested 2024-12-09T06:19:23,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163651 2024-12-09T06:19:23,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163651 2024-12-09T06:19:23,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163651) roll requested 2024-12-09T06:19:23,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163659 2024-12-09T06:19:23,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163659 2024-12-09T06:19:23,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163659) roll requested 2024-12-09T06:19:23,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163669 2024-12-09T06:19:23,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163669 2024-12-09T06:19:23,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163669) roll requested 2024-12-09T06:19:23,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163678 2024-12-09T06:19:23,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163678 2024-12-09T06:19:23,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163678) roll requested 2024-12-09T06:19:23,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163686 2024-12-09T06:19:23,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163686 2024-12-09T06:19:23,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163686) roll requested 2024-12-09T06:19:23,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163695 2024-12-09T06:19:23,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163695 2024-12-09T06:19:23,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163695) roll requested 2024-12-09T06:19:23,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163702 2024-12-09T06:19:23,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163702 2024-12-09T06:19:23,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163702) roll requested 2024-12-09T06:19:23,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163708 2024-12-09T06:19:23,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163708 2024-12-09T06:19:23,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163708) roll requested 2024-12-09T06:19:23,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163714 2024-12-09T06:19:23,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163714 2024-12-09T06:19:23,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163714) roll requested 2024-12-09T06:19:23,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163721 2024-12-09T06:19:23,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163721 2024-12-09T06:19:23,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163721) roll requested 2024-12-09T06:19:23,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163729 2024-12-09T06:19:23,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163729 2024-12-09T06:19:23,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163729) roll requested 2024-12-09T06:19:23,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163734 2024-12-09T06:19:23,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163734 2024-12-09T06:19:23,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163734) roll requested 2024-12-09T06:19:23,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163738 2024-12-09T06:19:23,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163738 2024-12-09T06:19:23,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163738) roll requested 2024-12-09T06:19:23,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163745 2024-12-09T06:19:23,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163745 2024-12-09T06:19:23,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163745) roll requested 2024-12-09T06:19:23,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163752 2024-12-09T06:19:23,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163752 2024-12-09T06:19:23,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163752) roll requested 2024-12-09T06:19:23,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163759 2024-12-09T06:19:23,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163759 2024-12-09T06:19:23,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163759) roll requested 2024-12-09T06:19:23,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163769 2024-12-09T06:19:23,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163769 2024-12-09T06:19:23,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163769) roll requested 2024-12-09T06:19:23,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163777 2024-12-09T06:19:23,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163777 2024-12-09T06:19:23,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163777) roll requested 2024-12-09T06:19:23,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163785 2024-12-09T06:19:23,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163785 2024-12-09T06:19:23,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163785) roll requested 2024-12-09T06:19:23,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163792 2024-12-09T06:19:23,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163792 2024-12-09T06:19:23,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163792) roll requested 2024-12-09T06:19:23,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163799 2024-12-09T06:19:23,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163799 2024-12-09T06:19:23,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163799) roll requested 2024-12-09T06:19:23,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163806 2024-12-09T06:19:23,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163806 2024-12-09T06:19:23,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163806) roll requested 2024-12-09T06:19:23,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163815 2024-12-09T06:19:23,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163815 2024-12-09T06:19:23,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163815) roll requested 2024-12-09T06:19:23,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163826 2024-12-09T06:19:23,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163826 2024-12-09T06:19:23,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163826) roll requested 2024-12-09T06:19:23,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163833 2024-12-09T06:19:23,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163833 2024-12-09T06:19:23,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163833) roll requested 2024-12-09T06:19:23,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163840 2024-12-09T06:19:23,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163840 2024-12-09T06:19:23,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163840) roll requested 2024-12-09T06:19:23,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163849 2024-12-09T06:19:23,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163849 2024-12-09T06:19:23,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163849) roll requested 2024-12-09T06:19:23,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163857 2024-12-09T06:19:23,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163857 2024-12-09T06:19:23,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163857) roll requested 2024-12-09T06:19:23,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163865 2024-12-09T06:19:23,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163865 2024-12-09T06:19:23,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163865) roll requested 2024-12-09T06:19:23,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163871 2024-12-09T06:19:23,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163871 2024-12-09T06:19:23,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163871) roll requested 2024-12-09T06:19:23,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163878 2024-12-09T06:19:23,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163878 2024-12-09T06:19:23,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163878) roll requested 2024-12-09T06:19:23,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163886 2024-12-09T06:19:23,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163886 2024-12-09T06:19:23,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163886) roll requested 2024-12-09T06:19:23,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163894 2024-12-09T06:19:23,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163894 2024-12-09T06:19:23,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163894) roll requested 2024-12-09T06:19:23,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163903 2024-12-09T06:19:23,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163903 2024-12-09T06:19:23,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163903) roll requested 2024-12-09T06:19:23,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163911 2024-12-09T06:19:23,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163911 2024-12-09T06:19:23,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163911) roll requested 2024-12-09T06:19:23,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163919 2024-12-09T06:19:23,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163919 2024-12-09T06:19:23,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163919) roll requested 2024-12-09T06:19:23,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163927 2024-12-09T06:19:23,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163927 2024-12-09T06:19:23,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163927) roll requested 2024-12-09T06:19:23,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163937 2024-12-09T06:19:23,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163937 2024-12-09T06:19:23,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163937) roll requested 2024-12-09T06:19:23,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163945 2024-12-09T06:19:23,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163945 2024-12-09T06:19:23,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163945) roll requested 2024-12-09T06:19:23,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163951 2024-12-09T06:19:23,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163951 2024-12-09T06:19:23,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163951) roll requested 2024-12-09T06:19:23,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163959 2024-12-09T06:19:23,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163959 2024-12-09T06:19:23,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163959) roll requested 2024-12-09T06:19:23,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163968 2024-12-09T06:19:23,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163968 2024-12-09T06:19:23,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163968) roll requested 2024-12-09T06:19:23,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163976 2024-12-09T06:19:23,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163976 2024-12-09T06:19:23,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163976) roll requested 2024-12-09T06:19:23,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163984 2024-12-09T06:19:23,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:23,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163984 2024-12-09T06:19:23,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:23,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:23,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:23,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:23,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163984) roll requested 2024-12-09T06:19:23,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725163993 2024-12-09T06:19:24,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163993 2024-12-09T06:19:24,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725163993) roll requested 2024-12-09T06:19:24,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164001 2024-12-09T06:19:24,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725163993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164001 2024-12-09T06:19:24,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164001) roll requested 2024-12-09T06:19:24,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164008 2024-12-09T06:19:24,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164008 2024-12-09T06:19:24,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164008) roll requested 2024-12-09T06:19:24,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164016 2024-12-09T06:19:24,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164016 2024-12-09T06:19:24,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164016) roll requested 2024-12-09T06:19:24,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164024 2024-12-09T06:19:24,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164024 2024-12-09T06:19:24,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164024) roll requested 2024-12-09T06:19:24,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164033 2024-12-09T06:19:24,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164033 2024-12-09T06:19:24,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164033) roll requested 2024-12-09T06:19:24,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164042 2024-12-09T06:19:24,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164042 2024-12-09T06:19:24,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164042) roll requested 2024-12-09T06:19:24,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164053 2024-12-09T06:19:24,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164053 2024-12-09T06:19:24,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164053) roll requested 2024-12-09T06:19:24,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164063 2024-12-09T06:19:24,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164063 2024-12-09T06:19:24,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164063) roll requested 2024-12-09T06:19:24,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164070 2024-12-09T06:19:24,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164070 2024-12-09T06:19:24,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164070) roll requested 2024-12-09T06:19:24,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164078 2024-12-09T06:19:24,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164078 2024-12-09T06:19:24,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164078) roll requested 2024-12-09T06:19:24,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164086 2024-12-09T06:19:24,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164086 2024-12-09T06:19:24,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164086) roll requested 2024-12-09T06:19:24,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164095 2024-12-09T06:19:24,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164095 2024-12-09T06:19:24,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164095) roll requested 2024-12-09T06:19:24,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164103 2024-12-09T06:19:24,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164103 2024-12-09T06:19:24,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164103) roll requested 2024-12-09T06:19:24,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164113 2024-12-09T06:19:24,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164113 2024-12-09T06:19:24,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164113) roll requested 2024-12-09T06:19:24,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164123 2024-12-09T06:19:24,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164123 2024-12-09T06:19:24,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164123) roll requested 2024-12-09T06:19:24,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164131 2024-12-09T06:19:24,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164131 2024-12-09T06:19:24,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164131) roll requested 2024-12-09T06:19:24,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164140 2024-12-09T06:19:24,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164140 2024-12-09T06:19:24,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164140) roll requested 2024-12-09T06:19:24,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164147 2024-12-09T06:19:24,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164147 2024-12-09T06:19:24,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164147) roll requested 2024-12-09T06:19:24,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164153 2024-12-09T06:19:24,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164153 2024-12-09T06:19:24,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164153) roll requested 2024-12-09T06:19:24,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164161 2024-12-09T06:19:24,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164161 2024-12-09T06:19:24,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164161) roll requested 2024-12-09T06:19:24,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164169 2024-12-09T06:19:24,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164169 2024-12-09T06:19:24,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164169) roll requested 2024-12-09T06:19:24,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164174 2024-12-09T06:19:24,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164174 2024-12-09T06:19:24,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164174) roll requested 2024-12-09T06:19:24,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164179 2024-12-09T06:19:24,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164179 2024-12-09T06:19:24,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164179) roll requested 2024-12-09T06:19:24,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164186 2024-12-09T06:19:24,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164186 2024-12-09T06:19:24,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164186) roll requested 2024-12-09T06:19:24,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164193 2024-12-09T06:19:24,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164193 2024-12-09T06:19:24,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164193) roll requested 2024-12-09T06:19:24,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164201 2024-12-09T06:19:24,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164201 2024-12-09T06:19:24,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164201) roll requested 2024-12-09T06:19:24,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164210 2024-12-09T06:19:24,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164210 2024-12-09T06:19:24,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164210) roll requested 2024-12-09T06:19:24,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164219 2024-12-09T06:19:24,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164219 2024-12-09T06:19:24,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164219) roll requested 2024-12-09T06:19:24,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164227 2024-12-09T06:19:24,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164227 2024-12-09T06:19:24,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164227) roll requested 2024-12-09T06:19:24,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164235 2024-12-09T06:19:24,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164235 2024-12-09T06:19:24,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164235) roll requested 2024-12-09T06:19:24,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164244 2024-12-09T06:19:24,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164244 2024-12-09T06:19:24,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164244) roll requested 2024-12-09T06:19:24,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164253 2024-12-09T06:19:24,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164253 2024-12-09T06:19:24,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164253) roll requested 2024-12-09T06:19:24,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164262 2024-12-09T06:19:24,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164262 2024-12-09T06:19:24,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164262) roll requested 2024-12-09T06:19:24,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164270 2024-12-09T06:19:24,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164270 2024-12-09T06:19:24,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164270) roll requested 2024-12-09T06:19:24,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164277 2024-12-09T06:19:24,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164277 2024-12-09T06:19:24,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164277) roll requested 2024-12-09T06:19:24,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164285 2024-12-09T06:19:24,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164285 2024-12-09T06:19:24,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164285) roll requested 2024-12-09T06:19:24,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164294 2024-12-09T06:19:24,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164294 2024-12-09T06:19:24,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164294) roll requested 2024-12-09T06:19:24,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164302 2024-12-09T06:19:24,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164302 2024-12-09T06:19:24,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164302) roll requested 2024-12-09T06:19:24,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164310 2024-12-09T06:19:24,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164310 2024-12-09T06:19:24,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164310) roll requested 2024-12-09T06:19:24,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164318 2024-12-09T06:19:24,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164318 2024-12-09T06:19:24,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164318) roll requested 2024-12-09T06:19:24,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164325 2024-12-09T06:19:24,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164325 2024-12-09T06:19:24,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164325) roll requested 2024-12-09T06:19:24,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164333 2024-12-09T06:19:24,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164333 2024-12-09T06:19:24,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164333) roll requested 2024-12-09T06:19:24,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164341 2024-12-09T06:19:24,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164341 2024-12-09T06:19:24,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164341) roll requested 2024-12-09T06:19:24,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164350 2024-12-09T06:19:24,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164350 2024-12-09T06:19:24,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164350) roll requested 2024-12-09T06:19:24,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164357 2024-12-09T06:19:24,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164357 2024-12-09T06:19:24,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164357) roll requested 2024-12-09T06:19:24,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164364 2024-12-09T06:19:24,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164364 2024-12-09T06:19:24,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164364) roll requested 2024-12-09T06:19:24,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164371 2024-12-09T06:19:24,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164371 2024-12-09T06:19:24,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164371) roll requested 2024-12-09T06:19:24,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164378 2024-12-09T06:19:24,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164378 2024-12-09T06:19:24,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164378) roll requested 2024-12-09T06:19:24,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164386 2024-12-09T06:19:24,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164386 2024-12-09T06:19:24,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164386) roll requested 2024-12-09T06:19:24,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164394 2024-12-09T06:19:24,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164394 2024-12-09T06:19:24,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164394) roll requested 2024-12-09T06:19:24,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164402 2024-12-09T06:19:24,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164402 2024-12-09T06:19:24,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164402) roll requested 2024-12-09T06:19:24,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164410 2024-12-09T06:19:24,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164410 2024-12-09T06:19:24,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164410) roll requested 2024-12-09T06:19:24,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164421 2024-12-09T06:19:24,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164421 2024-12-09T06:19:24,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164421) roll requested 2024-12-09T06:19:24,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164428 2024-12-09T06:19:24,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164428 2024-12-09T06:19:24,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164428) roll requested 2024-12-09T06:19:24,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164436 2024-12-09T06:19:24,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164436 2024-12-09T06:19:24,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164436) roll requested 2024-12-09T06:19:24,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164445 2024-12-09T06:19:24,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164445 2024-12-09T06:19:24,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164445) roll requested 2024-12-09T06:19:24,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164454 2024-12-09T06:19:24,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164454 2024-12-09T06:19:24,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164454) roll requested 2024-12-09T06:19:24,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164463 2024-12-09T06:19:24,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164463 2024-12-09T06:19:24,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164463) roll requested 2024-12-09T06:19:24,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164470 2024-12-09T06:19:24,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164470 2024-12-09T06:19:24,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164470) roll requested 2024-12-09T06:19:24,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164475 2024-12-09T06:19:24,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164475 2024-12-09T06:19:24,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164475) roll requested 2024-12-09T06:19:24,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164481 2024-12-09T06:19:24,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164481 2024-12-09T06:19:24,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164481) roll requested 2024-12-09T06:19:24,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164488 2024-12-09T06:19:24,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164488 2024-12-09T06:19:24,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164488) roll requested 2024-12-09T06:19:24,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164494 2024-12-09T06:19:24,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164494 2024-12-09T06:19:24,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164494) roll requested 2024-12-09T06:19:24,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164500 2024-12-09T06:19:24,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164500 2024-12-09T06:19:24,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164500) roll requested 2024-12-09T06:19:24,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164507 2024-12-09T06:19:24,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164507 2024-12-09T06:19:24,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164507) roll requested 2024-12-09T06:19:24,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164515 2024-12-09T06:19:24,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164515 2024-12-09T06:19:24,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164515) roll requested 2024-12-09T06:19:24,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164522 2024-12-09T06:19:24,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164522 2024-12-09T06:19:24,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164522) roll requested 2024-12-09T06:19:24,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164528 2024-12-09T06:19:24,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164528 2024-12-09T06:19:24,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164528) roll requested 2024-12-09T06:19:24,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164535 2024-12-09T06:19:24,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164535 2024-12-09T06:19:24,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164535) roll requested 2024-12-09T06:19:24,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164543 2024-12-09T06:19:24,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164543 2024-12-09T06:19:24,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164543) roll requested 2024-12-09T06:19:24,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164551 2024-12-09T06:19:24,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164551 2024-12-09T06:19:24,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164551) roll requested 2024-12-09T06:19:24,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164562 2024-12-09T06:19:24,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164562 2024-12-09T06:19:24,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164562) roll requested 2024-12-09T06:19:24,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164574 2024-12-09T06:19:24,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164574 2024-12-09T06:19:24,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164574) roll requested 2024-12-09T06:19:24,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164581 2024-12-09T06:19:24,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164581 2024-12-09T06:19:24,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164581) roll requested 2024-12-09T06:19:24,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164588 2024-12-09T06:19:24,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164588 2024-12-09T06:19:24,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164588) roll requested 2024-12-09T06:19:24,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164593 2024-12-09T06:19:24,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164593 2024-12-09T06:19:24,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164593) roll requested 2024-12-09T06:19:24,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164599 2024-12-09T06:19:24,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164599 2024-12-09T06:19:24,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164599) roll requested 2024-12-09T06:19:24,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164604 2024-12-09T06:19:24,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164604 2024-12-09T06:19:24,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164604) roll requested 2024-12-09T06:19:24,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164613 2024-12-09T06:19:24,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164613 2024-12-09T06:19:24,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164613) roll requested 2024-12-09T06:19:24,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164619 2024-12-09T06:19:24,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164619 2024-12-09T06:19:24,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164619) roll requested 2024-12-09T06:19:24,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164626 2024-12-09T06:19:24,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164626 2024-12-09T06:19:24,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164626) roll requested 2024-12-09T06:19:24,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164632 2024-12-09T06:19:24,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164632 2024-12-09T06:19:24,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164632) roll requested 2024-12-09T06:19:24,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164637 2024-12-09T06:19:24,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164637 2024-12-09T06:19:24,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164637) roll requested 2024-12-09T06:19:24,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164642 2024-12-09T06:19:24,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164642 2024-12-09T06:19:24,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164642) roll requested 2024-12-09T06:19:24,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164647 2024-12-09T06:19:24,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164647 2024-12-09T06:19:24,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164647) roll requested 2024-12-09T06:19:24,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164652 2024-12-09T06:19:24,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164652 2024-12-09T06:19:24,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164652) roll requested 2024-12-09T06:19:24,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164657 2024-12-09T06:19:24,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164657 2024-12-09T06:19:24,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164657) roll requested 2024-12-09T06:19:24,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164663 2024-12-09T06:19:24,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164663 2024-12-09T06:19:24,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164663) roll requested 2024-12-09T06:19:24,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164670 2024-12-09T06:19:24,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164670 2024-12-09T06:19:24,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164670) roll requested 2024-12-09T06:19:24,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164678 2024-12-09T06:19:24,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164678 2024-12-09T06:19:24,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164678) roll requested 2024-12-09T06:19:24,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164687 2024-12-09T06:19:24,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164687 2024-12-09T06:19:24,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164687) roll requested 2024-12-09T06:19:24,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164695 2024-12-09T06:19:24,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164695 2024-12-09T06:19:24,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164695) roll requested 2024-12-09T06:19:24,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164702 2024-12-09T06:19:24,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164702 2024-12-09T06:19:24,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164702) roll requested 2024-12-09T06:19:24,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164710 2024-12-09T06:19:24,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164710 2024-12-09T06:19:24,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164710) roll requested 2024-12-09T06:19:24,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164719 2024-12-09T06:19:24,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164719 2024-12-09T06:19:24,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164719) roll requested 2024-12-09T06:19:24,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164726 2024-12-09T06:19:24,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164726 2024-12-09T06:19:24,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164726) roll requested 2024-12-09T06:19:24,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164734 2024-12-09T06:19:24,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164734 2024-12-09T06:19:24,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164734) roll requested 2024-12-09T06:19:24,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164742 2024-12-09T06:19:24,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164742 2024-12-09T06:19:24,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164742) roll requested 2024-12-09T06:19:24,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164749 2024-12-09T06:19:24,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164749 2024-12-09T06:19:24,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164749) roll requested 2024-12-09T06:19:24,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164757 2024-12-09T06:19:24,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164757 2024-12-09T06:19:24,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164757) roll requested 2024-12-09T06:19:24,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164765 2024-12-09T06:19:24,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164765 2024-12-09T06:19:24,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164765) roll requested 2024-12-09T06:19:24,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164773 2024-12-09T06:19:24,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164773 2024-12-09T06:19:24,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164773) roll requested 2024-12-09T06:19:24,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164782 2024-12-09T06:19:24,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164782 2024-12-09T06:19:24,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164782) roll requested 2024-12-09T06:19:24,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164789 2024-12-09T06:19:24,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164789 2024-12-09T06:19:24,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164789) roll requested 2024-12-09T06:19:24,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164796 2024-12-09T06:19:24,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164796 2024-12-09T06:19:24,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164796) roll requested 2024-12-09T06:19:24,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164803 2024-12-09T06:19:24,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164803 2024-12-09T06:19:24,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164803) roll requested 2024-12-09T06:19:24,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164812 2024-12-09T06:19:24,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164812 2024-12-09T06:19:24,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164812) roll requested 2024-12-09T06:19:24,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164820 2024-12-09T06:19:24,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164820 2024-12-09T06:19:24,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164820) roll requested 2024-12-09T06:19:24,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164827 2024-12-09T06:19:24,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164827 2024-12-09T06:19:24,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164827) roll requested 2024-12-09T06:19:24,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164835 2024-12-09T06:19:24,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164835 2024-12-09T06:19:24,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164835) roll requested 2024-12-09T06:19:24,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164844 2024-12-09T06:19:24,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164844 2024-12-09T06:19:24,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164844) roll requested 2024-12-09T06:19:24,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164851 2024-12-09T06:19:24,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164851 2024-12-09T06:19:24,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164851) roll requested 2024-12-09T06:19:24,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164857 2024-12-09T06:19:24,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164857 2024-12-09T06:19:24,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164857) roll requested 2024-12-09T06:19:24,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164864 2024-12-09T06:19:24,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164864 2024-12-09T06:19:24,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164864) roll requested 2024-12-09T06:19:24,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164873 2024-12-09T06:19:24,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164873 2024-12-09T06:19:24,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164873) roll requested 2024-12-09T06:19:24,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164878 2024-12-09T06:19:24,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164878 2024-12-09T06:19:24,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164878) roll requested 2024-12-09T06:19:24,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164884 2024-12-09T06:19:24,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164884 2024-12-09T06:19:24,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164884) roll requested 2024-12-09T06:19:24,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164891 2024-12-09T06:19:24,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164891 2024-12-09T06:19:24,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164891) roll requested 2024-12-09T06:19:24,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164898 2024-12-09T06:19:24,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164898 2024-12-09T06:19:24,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164898) roll requested 2024-12-09T06:19:24,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164905 2024-12-09T06:19:24,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164905 2024-12-09T06:19:24,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164905) roll requested 2024-12-09T06:19:24,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164913 2024-12-09T06:19:24,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164913 2024-12-09T06:19:24,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164913) roll requested 2024-12-09T06:19:24,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164921 2024-12-09T06:19:24,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164921 2024-12-09T06:19:24,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164921) roll requested 2024-12-09T06:19:24,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164930 2024-12-09T06:19:24,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164930 2024-12-09T06:19:24,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164930) roll requested 2024-12-09T06:19:24,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164938 2024-12-09T06:19:24,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164938 2024-12-09T06:19:24,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164938) roll requested 2024-12-09T06:19:24,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164945 2024-12-09T06:19:24,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164945 2024-12-09T06:19:24,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164945) roll requested 2024-12-09T06:19:24,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164953 2024-12-09T06:19:24,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164953 2024-12-09T06:19:24,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164953) roll requested 2024-12-09T06:19:24,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164961 2024-12-09T06:19:24,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164961 2024-12-09T06:19:24,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164961) roll requested 2024-12-09T06:19:24,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164971 2024-12-09T06:19:24,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164971 2024-12-09T06:19:24,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164971) roll requested 2024-12-09T06:19:24,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164981 2024-12-09T06:19:24,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164981 2024-12-09T06:19:24,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164981) roll requested 2024-12-09T06:19:24,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164989 2024-12-09T06:19:24,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:24,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164989 2024-12-09T06:19:24,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:24,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:24,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:24,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:24,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164989) roll requested 2024-12-09T06:19:24,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725164995 2024-12-09T06:19:25,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164995 2024-12-09T06:19:25,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725164995) roll requested 2024-12-09T06:19:25,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165002 2024-12-09T06:19:25,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725164995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165002 2024-12-09T06:19:25,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165002) roll requested 2024-12-09T06:19:25,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165009 2024-12-09T06:19:25,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165009 2024-12-09T06:19:25,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165009) roll requested 2024-12-09T06:19:25,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165017 2024-12-09T06:19:25,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165017 2024-12-09T06:19:25,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165017) roll requested 2024-12-09T06:19:25,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165024 2024-12-09T06:19:25,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165024 2024-12-09T06:19:25,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165024) roll requested 2024-12-09T06:19:25,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165032 2024-12-09T06:19:25,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165032 2024-12-09T06:19:25,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165032) roll requested 2024-12-09T06:19:25,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165040 2024-12-09T06:19:25,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165040 2024-12-09T06:19:25,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165040) roll requested 2024-12-09T06:19:25,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165046 2024-12-09T06:19:25,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165046 2024-12-09T06:19:25,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165046) roll requested 2024-12-09T06:19:25,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165052 2024-12-09T06:19:25,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165052 2024-12-09T06:19:25,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165052) roll requested 2024-12-09T06:19:25,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165059 2024-12-09T06:19:25,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165059 2024-12-09T06:19:25,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165059) roll requested 2024-12-09T06:19:25,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165067 2024-12-09T06:19:25,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165067 2024-12-09T06:19:25,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165067) roll requested 2024-12-09T06:19:25,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165077 2024-12-09T06:19:25,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165077 2024-12-09T06:19:25,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165077) roll requested 2024-12-09T06:19:25,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165084 2024-12-09T06:19:25,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165084 2024-12-09T06:19:25,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165084) roll requested 2024-12-09T06:19:25,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165093 2024-12-09T06:19:25,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165093 2024-12-09T06:19:25,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165093) roll requested 2024-12-09T06:19:25,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165101 2024-12-09T06:19:25,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165101 2024-12-09T06:19:25,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165101) roll requested 2024-12-09T06:19:25,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165107 2024-12-09T06:19:25,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165107 2024-12-09T06:19:25,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165107) roll requested 2024-12-09T06:19:25,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165113 2024-12-09T06:19:25,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165113 2024-12-09T06:19:25,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165113) roll requested 2024-12-09T06:19:25,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165118 2024-12-09T06:19:25,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165118 2024-12-09T06:19:25,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165118) roll requested 2024-12-09T06:19:25,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165125 2024-12-09T06:19:25,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165125 2024-12-09T06:19:25,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165125) roll requested 2024-12-09T06:19:25,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165133 2024-12-09T06:19:25,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165133 2024-12-09T06:19:25,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165133) roll requested 2024-12-09T06:19:25,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165139 2024-12-09T06:19:25,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165139 2024-12-09T06:19:25,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165139) roll requested 2024-12-09T06:19:25,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165145 2024-12-09T06:19:25,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165145 2024-12-09T06:19:25,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165145) roll requested 2024-12-09T06:19:25,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165150 2024-12-09T06:19:25,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165150 2024-12-09T06:19:25,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165150) roll requested 2024-12-09T06:19:25,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165156 2024-12-09T06:19:25,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165156 2024-12-09T06:19:25,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165156) roll requested 2024-12-09T06:19:25,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165169 2024-12-09T06:19:25,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165169 2024-12-09T06:19:25,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165169) roll requested 2024-12-09T06:19:25,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165177 2024-12-09T06:19:25,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165177 2024-12-09T06:19:25,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165177) roll requested 2024-12-09T06:19:25,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165184 2024-12-09T06:19:25,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165184 2024-12-09T06:19:25,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165184) roll requested 2024-12-09T06:19:25,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165191 2024-12-09T06:19:25,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165191 2024-12-09T06:19:25,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165191) roll requested 2024-12-09T06:19:25,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165196 2024-12-09T06:19:25,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165196 2024-12-09T06:19:25,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165196) roll requested 2024-12-09T06:19:25,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165201 2024-12-09T06:19:25,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165201 2024-12-09T06:19:25,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165201) roll requested 2024-12-09T06:19:25,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165207 2024-12-09T06:19:25,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165207 2024-12-09T06:19:25,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165207) roll requested 2024-12-09T06:19:25,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165212 2024-12-09T06:19:25,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165212 2024-12-09T06:19:25,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165212) roll requested 2024-12-09T06:19:25,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165218 2024-12-09T06:19:25,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165218 2024-12-09T06:19:25,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165218) roll requested 2024-12-09T06:19:25,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165223 2024-12-09T06:19:25,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165223 2024-12-09T06:19:25,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165223) roll requested 2024-12-09T06:19:25,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165230 2024-12-09T06:19:25,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165230 2024-12-09T06:19:25,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165230) roll requested 2024-12-09T06:19:25,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165239 2024-12-09T06:19:25,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165239 2024-12-09T06:19:25,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165239) roll requested 2024-12-09T06:19:25,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165249 2024-12-09T06:19:25,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165249 2024-12-09T06:19:25,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165249) roll requested 2024-12-09T06:19:25,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165257 2024-12-09T06:19:25,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165257 2024-12-09T06:19:25,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165257) roll requested 2024-12-09T06:19:25,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165264 2024-12-09T06:19:25,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165264 2024-12-09T06:19:25,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165264) roll requested 2024-12-09T06:19:25,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165272 2024-12-09T06:19:25,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165272 2024-12-09T06:19:25,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165272) roll requested 2024-12-09T06:19:25,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165278 2024-12-09T06:19:25,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165278 2024-12-09T06:19:25,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165278) roll requested 2024-12-09T06:19:25,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165286 2024-12-09T06:19:25,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165286 2024-12-09T06:19:25,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165286) roll requested 2024-12-09T06:19:25,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165295 2024-12-09T06:19:25,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165295 2024-12-09T06:19:25,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165295) roll requested 2024-12-09T06:19:25,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165302 2024-12-09T06:19:25,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165302 2024-12-09T06:19:25,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165302) roll requested 2024-12-09T06:19:25,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165311 2024-12-09T06:19:25,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165311 2024-12-09T06:19:25,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165311) roll requested 2024-12-09T06:19:25,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165323 2024-12-09T06:19:25,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165323 2024-12-09T06:19:25,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165323) roll requested 2024-12-09T06:19:25,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165329 2024-12-09T06:19:25,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165329 2024-12-09T06:19:25,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165329) roll requested 2024-12-09T06:19:25,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165334 2024-12-09T06:19:25,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165334 2024-12-09T06:19:25,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165334) roll requested 2024-12-09T06:19:25,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165339 2024-12-09T06:19:25,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165339 2024-12-09T06:19:25,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165339) roll requested 2024-12-09T06:19:25,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165346 2024-12-09T06:19:25,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165346 2024-12-09T06:19:25,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165346) roll requested 2024-12-09T06:19:25,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165353 2024-12-09T06:19:25,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165353 2024-12-09T06:19:25,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165353) roll requested 2024-12-09T06:19:25,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165360 2024-12-09T06:19:25,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165360 2024-12-09T06:19:25,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165360) roll requested 2024-12-09T06:19:25,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165369 2024-12-09T06:19:25,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165369 2024-12-09T06:19:25,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165369) roll requested 2024-12-09T06:19:25,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165377 2024-12-09T06:19:25,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165377 2024-12-09T06:19:25,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165377) roll requested 2024-12-09T06:19:25,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165387 2024-12-09T06:19:25,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165387 2024-12-09T06:19:25,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165387) roll requested 2024-12-09T06:19:25,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165398 2024-12-09T06:19:25,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165398 2024-12-09T06:19:25,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165398) roll requested 2024-12-09T06:19:25,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165409 2024-12-09T06:19:25,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165409 2024-12-09T06:19:25,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165409) roll requested 2024-12-09T06:19:25,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165416 2024-12-09T06:19:25,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165416 2024-12-09T06:19:25,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165416) roll requested 2024-12-09T06:19:25,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165423 2024-12-09T06:19:25,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165423 2024-12-09T06:19:25,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165423) roll requested 2024-12-09T06:19:25,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165431 2024-12-09T06:19:25,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165431 2024-12-09T06:19:25,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165431) roll requested 2024-12-09T06:19:25,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165440 2024-12-09T06:19:25,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165440 2024-12-09T06:19:25,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165440) roll requested 2024-12-09T06:19:25,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165449 2024-12-09T06:19:25,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165449 2024-12-09T06:19:25,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165449) roll requested 2024-12-09T06:19:25,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165456 2024-12-09T06:19:25,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165456 2024-12-09T06:19:25,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165456) roll requested 2024-12-09T06:19:25,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165469 2024-12-09T06:19:25,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165469 2024-12-09T06:19:25,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165469) roll requested 2024-12-09T06:19:25,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165477 2024-12-09T06:19:25,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165477 2024-12-09T06:19:25,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165477) roll requested 2024-12-09T06:19:25,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165486 2024-12-09T06:19:25,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165486 2024-12-09T06:19:25,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165486) roll requested 2024-12-09T06:19:25,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165496 2024-12-09T06:19:25,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165496 2024-12-09T06:19:25,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165496) roll requested 2024-12-09T06:19:25,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165502 2024-12-09T06:19:25,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165502 2024-12-09T06:19:25,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165502) roll requested 2024-12-09T06:19:25,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165508 2024-12-09T06:19:25,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165508 2024-12-09T06:19:25,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165508) roll requested 2024-12-09T06:19:25,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165515 2024-12-09T06:19:25,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165515 2024-12-09T06:19:25,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165515) roll requested 2024-12-09T06:19:25,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165522 2024-12-09T06:19:25,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165522 2024-12-09T06:19:25,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165522) roll requested 2024-12-09T06:19:25,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165528 2024-12-09T06:19:25,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165528 2024-12-09T06:19:25,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165528) roll requested 2024-12-09T06:19:25,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165540 2024-12-09T06:19:25,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165540 2024-12-09T06:19:25,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165540) roll requested 2024-12-09T06:19:25,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165547 2024-12-09T06:19:25,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165547 2024-12-09T06:19:25,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165547) roll requested 2024-12-09T06:19:25,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165553 2024-12-09T06:19:25,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165553 2024-12-09T06:19:25,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165553) roll requested 2024-12-09T06:19:25,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165561 2024-12-09T06:19:25,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165561 2024-12-09T06:19:25,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165561) roll requested 2024-12-09T06:19:25,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165569 2024-12-09T06:19:25,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165569 2024-12-09T06:19:25,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165569) roll requested 2024-12-09T06:19:25,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165577 2024-12-09T06:19:25,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165577 2024-12-09T06:19:25,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165577) roll requested 2024-12-09T06:19:25,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165585 2024-12-09T06:19:25,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165585 2024-12-09T06:19:25,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165585) roll requested 2024-12-09T06:19:25,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165592 2024-12-09T06:19:25,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165592 2024-12-09T06:19:25,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165592) roll requested 2024-12-09T06:19:25,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165597 2024-12-09T06:19:25,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165597 2024-12-09T06:19:25,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165597) roll requested 2024-12-09T06:19:25,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165603 2024-12-09T06:19:25,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165603 2024-12-09T06:19:25,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165603) roll requested 2024-12-09T06:19:25,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165610 2024-12-09T06:19:25,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165610 2024-12-09T06:19:25,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165610) roll requested 2024-12-09T06:19:25,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165616 2024-12-09T06:19:25,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165616 2024-12-09T06:19:25,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165616) roll requested 2024-12-09T06:19:25,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165622 2024-12-09T06:19:25,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165622 2024-12-09T06:19:25,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165622) roll requested 2024-12-09T06:19:25,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165628 2024-12-09T06:19:25,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165628 2024-12-09T06:19:25,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165628) roll requested 2024-12-09T06:19:25,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165634 2024-12-09T06:19:25,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165634 2024-12-09T06:19:25,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165634) roll requested 2024-12-09T06:19:25,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165640 2024-12-09T06:19:25,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165640 2024-12-09T06:19:25,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165640) roll requested 2024-12-09T06:19:25,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165647 2024-12-09T06:19:25,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165647 2024-12-09T06:19:25,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165647) roll requested 2024-12-09T06:19:25,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165653 2024-12-09T06:19:25,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165653 2024-12-09T06:19:25,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165653) roll requested 2024-12-09T06:19:25,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165660 2024-12-09T06:19:25,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165660 2024-12-09T06:19:25,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165660) roll requested 2024-12-09T06:19:25,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165667 2024-12-09T06:19:25,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165667 2024-12-09T06:19:25,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165667) roll requested 2024-12-09T06:19:25,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165673 2024-12-09T06:19:25,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165673 2024-12-09T06:19:25,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165673) roll requested 2024-12-09T06:19:25,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165681 2024-12-09T06:19:25,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165681 2024-12-09T06:19:25,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165681) roll requested 2024-12-09T06:19:25,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165687 2024-12-09T06:19:25,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165687 2024-12-09T06:19:25,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165687) roll requested 2024-12-09T06:19:25,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165693 2024-12-09T06:19:25,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165693 2024-12-09T06:19:25,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165693) roll requested 2024-12-09T06:19:25,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165699 2024-12-09T06:19:25,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165699 2024-12-09T06:19:25,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165699) roll requested 2024-12-09T06:19:25,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165705 2024-12-09T06:19:25,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165705 2024-12-09T06:19:25,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165705) roll requested 2024-12-09T06:19:25,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165710 2024-12-09T06:19:25,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165710 2024-12-09T06:19:25,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165710) roll requested 2024-12-09T06:19:25,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165719 2024-12-09T06:19:25,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165719 2024-12-09T06:19:25,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165719) roll requested 2024-12-09T06:19:25,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165724 2024-12-09T06:19:25,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165724 2024-12-09T06:19:25,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165724) roll requested 2024-12-09T06:19:25,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165729 2024-12-09T06:19:25,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165729 2024-12-09T06:19:25,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165729) roll requested 2024-12-09T06:19:25,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165734 2024-12-09T06:19:25,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165734 2024-12-09T06:19:25,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165734) roll requested 2024-12-09T06:19:25,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165740 2024-12-09T06:19:25,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165740 2024-12-09T06:19:25,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165740) roll requested 2024-12-09T06:19:25,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165746 2024-12-09T06:19:25,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165746 2024-12-09T06:19:25,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165746) roll requested 2024-12-09T06:19:25,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165755 2024-12-09T06:19:25,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165755 2024-12-09T06:19:25,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165755) roll requested 2024-12-09T06:19:25,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165761 2024-12-09T06:19:25,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165761 2024-12-09T06:19:25,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165761) roll requested 2024-12-09T06:19:25,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165769 2024-12-09T06:19:25,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165769 2024-12-09T06:19:25,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165769) roll requested 2024-12-09T06:19:25,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165775 2024-12-09T06:19:25,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165775 2024-12-09T06:19:25,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165775) roll requested 2024-12-09T06:19:25,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165782 2024-12-09T06:19:25,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165782 2024-12-09T06:19:25,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165782) roll requested 2024-12-09T06:19:25,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165788 2024-12-09T06:19:25,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165788 2024-12-09T06:19:25,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165788) roll requested 2024-12-09T06:19:25,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165796 2024-12-09T06:19:25,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165796 2024-12-09T06:19:25,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165796) roll requested 2024-12-09T06:19:25,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165804 2024-12-09T06:19:25,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165804 2024-12-09T06:19:25,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165804) roll requested 2024-12-09T06:19:25,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165812 2024-12-09T06:19:25,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165812 2024-12-09T06:19:25,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165812) roll requested 2024-12-09T06:19:25,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165821 2024-12-09T06:19:25,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165821 2024-12-09T06:19:25,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165821) roll requested 2024-12-09T06:19:25,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165829 2024-12-09T06:19:25,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165829 2024-12-09T06:19:25,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165829) roll requested 2024-12-09T06:19:25,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165838 2024-12-09T06:19:25,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165838 2024-12-09T06:19:25,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165838) roll requested 2024-12-09T06:19:25,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165846 2024-12-09T06:19:25,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165846 2024-12-09T06:19:25,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165846) roll requested 2024-12-09T06:19:25,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165854 2024-12-09T06:19:25,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165854 2024-12-09T06:19:25,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165854) roll requested 2024-12-09T06:19:25,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165862 2024-12-09T06:19:25,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165862 2024-12-09T06:19:25,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165862) roll requested 2024-12-09T06:19:25,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165870 2024-12-09T06:19:25,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165870 2024-12-09T06:19:25,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165870) roll requested 2024-12-09T06:19:25,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165879 2024-12-09T06:19:25,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165879 2024-12-09T06:19:25,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165879) roll requested 2024-12-09T06:19:25,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165886 2024-12-09T06:19:25,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165886 2024-12-09T06:19:25,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165886) roll requested 2024-12-09T06:19:25,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165894 2024-12-09T06:19:25,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165894 2024-12-09T06:19:25,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165894) roll requested 2024-12-09T06:19:25,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165902 2024-12-09T06:19:25,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165902 2024-12-09T06:19:25,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165902) roll requested 2024-12-09T06:19:25,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165909 2024-12-09T06:19:25,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165909 2024-12-09T06:19:25,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165909) roll requested 2024-12-09T06:19:25,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165916 2024-12-09T06:19:25,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165916 2024-12-09T06:19:25,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165916) roll requested 2024-12-09T06:19:25,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165924 2024-12-09T06:19:25,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165924 2024-12-09T06:19:25,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165924) roll requested 2024-12-09T06:19:25,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165935 2024-12-09T06:19:25,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165935 2024-12-09T06:19:25,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165935) roll requested 2024-12-09T06:19:25,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165946 2024-12-09T06:19:25,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165946 2024-12-09T06:19:25,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165946) roll requested 2024-12-09T06:19:25,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165955 2024-12-09T06:19:25,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165955 2024-12-09T06:19:25,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165955) roll requested 2024-12-09T06:19:25,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165960 2024-12-09T06:19:25,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165960 2024-12-09T06:19:25,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165960) roll requested 2024-12-09T06:19:25,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165965 2024-12-09T06:19:25,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165965 2024-12-09T06:19:25,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165965) roll requested 2024-12-09T06:19:25,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165970 2024-12-09T06:19:25,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165970 2024-12-09T06:19:25,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165970) roll requested 2024-12-09T06:19:25,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165975 2024-12-09T06:19:25,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165975 2024-12-09T06:19:25,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165975) roll requested 2024-12-09T06:19:25,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165982 2024-12-09T06:19:25,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165982 2024-12-09T06:19:25,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165982) roll requested 2024-12-09T06:19:25,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165989 2024-12-09T06:19:25,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:25,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165989 2024-12-09T06:19:25,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:25,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:25,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:25,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:25,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165989) roll requested 2024-12-09T06:19:25,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725165997 2024-12-09T06:19:26,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165997 2024-12-09T06:19:26,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725165997) roll requested 2024-12-09T06:19:26,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166005 2024-12-09T06:19:26,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725165997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166005 2024-12-09T06:19:26,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166005) roll requested 2024-12-09T06:19:26,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166012 2024-12-09T06:19:26,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166012 2024-12-09T06:19:26,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166012) roll requested 2024-12-09T06:19:26,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166020 2024-12-09T06:19:26,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166020 2024-12-09T06:19:26,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166020) roll requested 2024-12-09T06:19:26,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166029 2024-12-09T06:19:26,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166029 2024-12-09T06:19:26,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166029) roll requested 2024-12-09T06:19:26,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166035 2024-12-09T06:19:26,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166035 2024-12-09T06:19:26,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166035) roll requested 2024-12-09T06:19:26,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166043 2024-12-09T06:19:26,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166043 2024-12-09T06:19:26,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166043) roll requested 2024-12-09T06:19:26,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166053 2024-12-09T06:19:26,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166053 2024-12-09T06:19:26,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166053) roll requested 2024-12-09T06:19:26,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166065 2024-12-09T06:19:26,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166065 2024-12-09T06:19:26,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166065) roll requested 2024-12-09T06:19:26,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166072 2024-12-09T06:19:26,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166072 2024-12-09T06:19:26,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166072) roll requested 2024-12-09T06:19:26,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166083 2024-12-09T06:19:26,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166083 2024-12-09T06:19:26,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166083) roll requested 2024-12-09T06:19:26,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166091 2024-12-09T06:19:26,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166091 2024-12-09T06:19:26,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166091) roll requested 2024-12-09T06:19:26,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166099 2024-12-09T06:19:26,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166099 2024-12-09T06:19:26,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166099) roll requested 2024-12-09T06:19:26,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166107 2024-12-09T06:19:26,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166107 2024-12-09T06:19:26,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166107) roll requested 2024-12-09T06:19:26,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166117 2024-12-09T06:19:26,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166117 2024-12-09T06:19:26,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166117) roll requested 2024-12-09T06:19:26,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166123 2024-12-09T06:19:26,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166123 2024-12-09T06:19:26,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166123) roll requested 2024-12-09T06:19:26,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166130 2024-12-09T06:19:26,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166130 2024-12-09T06:19:26,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166130) roll requested 2024-12-09T06:19:26,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166138 2024-12-09T06:19:26,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166138 2024-12-09T06:19:26,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166138) roll requested 2024-12-09T06:19:26,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166146 2024-12-09T06:19:26,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166146 2024-12-09T06:19:26,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166146) roll requested 2024-12-09T06:19:26,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166155 2024-12-09T06:19:26,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166155 2024-12-09T06:19:26,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166155) roll requested 2024-12-09T06:19:26,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166161 2024-12-09T06:19:26,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166161 2024-12-09T06:19:26,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166161) roll requested 2024-12-09T06:19:26,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166168 2024-12-09T06:19:26,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166168 2024-12-09T06:19:26,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166168) roll requested 2024-12-09T06:19:26,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166177 2024-12-09T06:19:26,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166177 2024-12-09T06:19:26,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166177) roll requested 2024-12-09T06:19:26,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166185 2024-12-09T06:19:26,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166185 2024-12-09T06:19:26,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166185) roll requested 2024-12-09T06:19:26,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166190 2024-12-09T06:19:26,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166190 2024-12-09T06:19:26,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166190) roll requested 2024-12-09T06:19:26,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166195 2024-12-09T06:19:26,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166195 2024-12-09T06:19:26,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166195) roll requested 2024-12-09T06:19:26,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166200 2024-12-09T06:19:26,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166200 2024-12-09T06:19:26,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166200) roll requested 2024-12-09T06:19:26,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166206 2024-12-09T06:19:26,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166206 2024-12-09T06:19:26,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166206) roll requested 2024-12-09T06:19:26,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166211 2024-12-09T06:19:26,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166211 2024-12-09T06:19:26,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166211) roll requested 2024-12-09T06:19:26,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166217 2024-12-09T06:19:26,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166217 2024-12-09T06:19:26,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166217) roll requested 2024-12-09T06:19:26,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166223 2024-12-09T06:19:26,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166223 2024-12-09T06:19:26,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166223) roll requested 2024-12-09T06:19:26,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166230 2024-12-09T06:19:26,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166230 2024-12-09T06:19:26,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166230) roll requested 2024-12-09T06:19:26,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166237 2024-12-09T06:19:26,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166237 2024-12-09T06:19:26,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166237) roll requested 2024-12-09T06:19:26,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166245 2024-12-09T06:19:26,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166245 2024-12-09T06:19:26,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166245) roll requested 2024-12-09T06:19:26,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166251 2024-12-09T06:19:26,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166251 2024-12-09T06:19:26,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166251) roll requested 2024-12-09T06:19:26,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166257 2024-12-09T06:19:26,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166257 2024-12-09T06:19:26,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166257) roll requested 2024-12-09T06:19:26,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166265 2024-12-09T06:19:26,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166265 2024-12-09T06:19:26,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166265) roll requested 2024-12-09T06:19:26,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166273 2024-12-09T06:19:26,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166273 2024-12-09T06:19:26,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166273) roll requested 2024-12-09T06:19:26,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166281 2024-12-09T06:19:26,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166281 2024-12-09T06:19:26,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166281) roll requested 2024-12-09T06:19:26,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166289 2024-12-09T06:19:26,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166289 2024-12-09T06:19:26,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166289) roll requested 2024-12-09T06:19:26,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166296 2024-12-09T06:19:26,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166296 2024-12-09T06:19:26,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166296) roll requested 2024-12-09T06:19:26,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166302 2024-12-09T06:19:26,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166302 2024-12-09T06:19:26,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166302) roll requested 2024-12-09T06:19:26,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166310 2024-12-09T06:19:26,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166310 2024-12-09T06:19:26,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166310) roll requested 2024-12-09T06:19:26,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166317 2024-12-09T06:19:26,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166317 2024-12-09T06:19:26,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166317) roll requested 2024-12-09T06:19:26,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166325 2024-12-09T06:19:26,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166325 2024-12-09T06:19:26,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166325) roll requested 2024-12-09T06:19:26,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166331 2024-12-09T06:19:26,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166331 2024-12-09T06:19:26,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166331) roll requested 2024-12-09T06:19:26,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166340 2024-12-09T06:19:26,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166340 2024-12-09T06:19:26,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166340) roll requested 2024-12-09T06:19:26,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166348 2024-12-09T06:19:26,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166348 2024-12-09T06:19:26,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166348) roll requested 2024-12-09T06:19:26,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166356 2024-12-09T06:19:26,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166356 2024-12-09T06:19:26,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166356) roll requested 2024-12-09T06:19:26,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166365 2024-12-09T06:19:26,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166365 2024-12-09T06:19:26,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166365) roll requested 2024-12-09T06:19:26,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166373 2024-12-09T06:19:26,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166373 2024-12-09T06:19:26,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166373) roll requested 2024-12-09T06:19:26,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166380 2024-12-09T06:19:26,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166380 2024-12-09T06:19:26,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166380) roll requested 2024-12-09T06:19:26,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166388 2024-12-09T06:19:26,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166388 2024-12-09T06:19:26,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166388) roll requested 2024-12-09T06:19:26,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166396 2024-12-09T06:19:26,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166396 2024-12-09T06:19:26,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166396) roll requested 2024-12-09T06:19:26,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166405 2024-12-09T06:19:26,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166405 2024-12-09T06:19:26,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166405) roll requested 2024-12-09T06:19:26,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166416 2024-12-09T06:19:26,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166416 2024-12-09T06:19:26,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166416) roll requested 2024-12-09T06:19:26,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166425 2024-12-09T06:19:26,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166425 2024-12-09T06:19:26,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166425) roll requested 2024-12-09T06:19:26,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166435 2024-12-09T06:19:26,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166435 2024-12-09T06:19:26,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166435) roll requested 2024-12-09T06:19:26,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166440 2024-12-09T06:19:26,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166440 2024-12-09T06:19:26,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166440) roll requested 2024-12-09T06:19:26,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166446 2024-12-09T06:19:26,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166446 2024-12-09T06:19:26,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166446) roll requested 2024-12-09T06:19:26,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166452 2024-12-09T06:19:26,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166452 2024-12-09T06:19:26,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166452) roll requested 2024-12-09T06:19:26,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166459 2024-12-09T06:19:26,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166459 2024-12-09T06:19:26,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166459) roll requested 2024-12-09T06:19:26,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166466 2024-12-09T06:19:26,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166466 2024-12-09T06:19:26,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166466) roll requested 2024-12-09T06:19:26,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166473 2024-12-09T06:19:26,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166473 2024-12-09T06:19:26,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166473) roll requested 2024-12-09T06:19:26,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166481 2024-12-09T06:19:26,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166481 2024-12-09T06:19:26,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166481) roll requested 2024-12-09T06:19:26,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166490 2024-12-09T06:19:26,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166490 2024-12-09T06:19:26,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166490) roll requested 2024-12-09T06:19:26,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166497 2024-12-09T06:19:26,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166497 2024-12-09T06:19:26,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166497) roll requested 2024-12-09T06:19:26,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166504 2024-12-09T06:19:26,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166504 2024-12-09T06:19:26,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166504) roll requested 2024-12-09T06:19:26,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166510 2024-12-09T06:19:26,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166510 2024-12-09T06:19:26,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166510) roll requested 2024-12-09T06:19:26,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166518 2024-12-09T06:19:26,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166518 2024-12-09T06:19:26,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166518) roll requested 2024-12-09T06:19:26,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166528 2024-12-09T06:19:26,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166528 2024-12-09T06:19:26,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166528) roll requested 2024-12-09T06:19:26,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166539 2024-12-09T06:19:26,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166539 2024-12-09T06:19:26,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166539) roll requested 2024-12-09T06:19:26,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166544 2024-12-09T06:19:26,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166544 2024-12-09T06:19:26,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166544) roll requested 2024-12-09T06:19:26,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166549 2024-12-09T06:19:26,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166549 2024-12-09T06:19:26,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166549) roll requested 2024-12-09T06:19:26,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166555 2024-12-09T06:19:26,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166555 2024-12-09T06:19:26,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166555) roll requested 2024-12-09T06:19:26,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166561 2024-12-09T06:19:26,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166561 2024-12-09T06:19:26,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166561) roll requested 2024-12-09T06:19:26,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166569 2024-12-09T06:19:26,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166569 2024-12-09T06:19:26,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166569) roll requested 2024-12-09T06:19:26,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166577 2024-12-09T06:19:26,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166577 2024-12-09T06:19:26,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166577) roll requested 2024-12-09T06:19:26,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166584 2024-12-09T06:19:26,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166584 2024-12-09T06:19:26,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166584) roll requested 2024-12-09T06:19:26,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166593 2024-12-09T06:19:26,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166593 2024-12-09T06:19:26,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166593) roll requested 2024-12-09T06:19:26,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166600 2024-12-09T06:19:26,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166600 2024-12-09T06:19:26,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166600) roll requested 2024-12-09T06:19:26,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166609 2024-12-09T06:19:26,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166609 2024-12-09T06:19:26,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166609) roll requested 2024-12-09T06:19:26,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166617 2024-12-09T06:19:26,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166617 2024-12-09T06:19:26,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166617) roll requested 2024-12-09T06:19:26,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166627 2024-12-09T06:19:26,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166627 2024-12-09T06:19:26,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166627) roll requested 2024-12-09T06:19:26,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166637 2024-12-09T06:19:26,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166637 2024-12-09T06:19:26,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166637) roll requested 2024-12-09T06:19:26,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166645 2024-12-09T06:19:26,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166645 2024-12-09T06:19:26,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166645) roll requested 2024-12-09T06:19:26,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166655 2024-12-09T06:19:26,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166655 2024-12-09T06:19:26,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166655) roll requested 2024-12-09T06:19:26,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166668 2024-12-09T06:19:26,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166668 2024-12-09T06:19:26,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166668) roll requested 2024-12-09T06:19:26,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166675 2024-12-09T06:19:26,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166675 2024-12-09T06:19:26,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166675) roll requested 2024-12-09T06:19:26,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166681 2024-12-09T06:19:26,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166681 2024-12-09T06:19:26,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166681) roll requested 2024-12-09T06:19:26,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166690 2024-12-09T06:19:26,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166690 2024-12-09T06:19:26,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166690) roll requested 2024-12-09T06:19:26,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166698 2024-12-09T06:19:26,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166698 2024-12-09T06:19:26,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166698) roll requested 2024-12-09T06:19:26,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166707 2024-12-09T06:19:26,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166707 2024-12-09T06:19:26,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166707) roll requested 2024-12-09T06:19:26,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166717 2024-12-09T06:19:26,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166717 2024-12-09T06:19:26,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166717) roll requested 2024-12-09T06:19:26,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166725 2024-12-09T06:19:26,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166725 2024-12-09T06:19:26,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166725) roll requested 2024-12-09T06:19:26,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166733 2024-12-09T06:19:26,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166733 2024-12-09T06:19:26,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166733) roll requested 2024-12-09T06:19:26,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166740 2024-12-09T06:19:26,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166740 2024-12-09T06:19:26,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166740) roll requested 2024-12-09T06:19:26,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166750 2024-12-09T06:19:26,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166750 2024-12-09T06:19:26,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166750) roll requested 2024-12-09T06:19:26,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166755 2024-12-09T06:19:26,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166755 2024-12-09T06:19:26,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166755) roll requested 2024-12-09T06:19:26,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166760 2024-12-09T06:19:26,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166760 2024-12-09T06:19:26,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166760) roll requested 2024-12-09T06:19:26,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166769 2024-12-09T06:19:26,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166769 2024-12-09T06:19:26,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166769) roll requested 2024-12-09T06:19:26,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166776 2024-12-09T06:19:26,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166776 2024-12-09T06:19:26,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166776) roll requested 2024-12-09T06:19:26,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166784 2024-12-09T06:19:26,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166784 2024-12-09T06:19:26,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166784) roll requested 2024-12-09T06:19:26,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166794 2024-12-09T06:19:26,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166794 2024-12-09T06:19:26,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166794) roll requested 2024-12-09T06:19:26,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166804 2024-12-09T06:19:26,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166804 2024-12-09T06:19:26,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166804) roll requested 2024-12-09T06:19:26,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166813 2024-12-09T06:19:26,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166813 2024-12-09T06:19:26,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166813) roll requested 2024-12-09T06:19:26,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166823 2024-12-09T06:19:26,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166823 2024-12-09T06:19:26,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166823) roll requested 2024-12-09T06:19:26,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166831 2024-12-09T06:19:26,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166831 2024-12-09T06:19:26,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166831) roll requested 2024-12-09T06:19:26,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166840 2024-12-09T06:19:26,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166840 2024-12-09T06:19:26,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166840) roll requested 2024-12-09T06:19:26,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166849 2024-12-09T06:19:26,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166849 2024-12-09T06:19:26,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166849) roll requested 2024-12-09T06:19:26,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166856 2024-12-09T06:19:26,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166856 2024-12-09T06:19:26,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166856) roll requested 2024-12-09T06:19:26,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166864 2024-12-09T06:19:26,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166864 2024-12-09T06:19:26,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166864) roll requested 2024-12-09T06:19:26,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166872 2024-12-09T06:19:26,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166872 2024-12-09T06:19:26,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166872) roll requested 2024-12-09T06:19:26,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166879 2024-12-09T06:19:26,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166879 2024-12-09T06:19:26,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166879) roll requested 2024-12-09T06:19:26,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166886 2024-12-09T06:19:26,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166886 2024-12-09T06:19:26,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166886) roll requested 2024-12-09T06:19:26,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166894 2024-12-09T06:19:26,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166894 2024-12-09T06:19:26,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166894) roll requested 2024-12-09T06:19:26,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166902 2024-12-09T06:19:26,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166902 2024-12-09T06:19:26,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166902) roll requested 2024-12-09T06:19:26,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166910 2024-12-09T06:19:26,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166910 2024-12-09T06:19:26,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166910) roll requested 2024-12-09T06:19:26,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166917 2024-12-09T06:19:26,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166917 2024-12-09T06:19:26,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166917) roll requested 2024-12-09T06:19:26,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166925 2024-12-09T06:19:26,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166925 2024-12-09T06:19:26,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166925) roll requested 2024-12-09T06:19:26,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166933 2024-12-09T06:19:26,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166933 2024-12-09T06:19:26,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166933) roll requested 2024-12-09T06:19:26,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166939 2024-12-09T06:19:26,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166939 2024-12-09T06:19:26,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166939) roll requested 2024-12-09T06:19:26,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166946 2024-12-09T06:19:26,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166946 2024-12-09T06:19:26,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166946) roll requested 2024-12-09T06:19:26,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166954 2024-12-09T06:19:26,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166954 2024-12-09T06:19:26,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166954) roll requested 2024-12-09T06:19:26,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166962 2024-12-09T06:19:26,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166962 2024-12-09T06:19:26,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166962) roll requested 2024-12-09T06:19:26,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166973 2024-12-09T06:19:26,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166973 2024-12-09T06:19:26,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166973) roll requested 2024-12-09T06:19:26,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166981 2024-12-09T06:19:26,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166981 2024-12-09T06:19:26,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166981) roll requested 2024-12-09T06:19:26,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166986 2024-12-09T06:19:26,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166986 2024-12-09T06:19:26,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166986) roll requested 2024-12-09T06:19:26,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166992 2024-12-09T06:19:26,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:26,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166992 2024-12-09T06:19:26,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:26,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:26,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:26,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:26,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166992) roll requested 2024-12-09T06:19:26,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725166998 2024-12-09T06:19:27,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166998 2024-12-09T06:19:27,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725166998) roll requested 2024-12-09T06:19:27,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167003 2024-12-09T06:19:27,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725166998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167003 2024-12-09T06:19:27,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167003) roll requested 2024-12-09T06:19:27,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167008 2024-12-09T06:19:27,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167008 2024-12-09T06:19:27,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167008) roll requested 2024-12-09T06:19:27,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167014 2024-12-09T06:19:27,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167014 2024-12-09T06:19:27,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167014) roll requested 2024-12-09T06:19:27,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167019 2024-12-09T06:19:27,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167019 2024-12-09T06:19:27,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167014 is not closed yet, will try archiving it next time 2024-12-09T06:19:27,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167019) roll requested 2024-12-09T06:19:27,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167025 2024-12-09T06:19:27,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167025 2024-12-09T06:19:27,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167025) roll requested 2024-12-09T06:19:27,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167032 2024-12-09T06:19:27,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167032 2024-12-09T06:19:27,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167032) roll requested 2024-12-09T06:19:27,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167039 2024-12-09T06:19:27,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167039 2024-12-09T06:19:27,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167039) roll requested 2024-12-09T06:19:27,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167047 2024-12-09T06:19:27,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167047 2024-12-09T06:19:27,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167047) roll requested 2024-12-09T06:19:27,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167056 2024-12-09T06:19:27,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167056 2024-12-09T06:19:27,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167056) roll requested 2024-12-09T06:19:27,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167065 2024-12-09T06:19:27,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167065 2024-12-09T06:19:27,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167065) roll requested 2024-12-09T06:19:27,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167074 2024-12-09T06:19:27,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167074 2024-12-09T06:19:27,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167074) roll requested 2024-12-09T06:19:27,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167083 2024-12-09T06:19:27,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167083 2024-12-09T06:19:27,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167083) roll requested 2024-12-09T06:19:27,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167091 2024-12-09T06:19:27,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167091 2024-12-09T06:19:27,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167091) roll requested 2024-12-09T06:19:27,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167098 2024-12-09T06:19:27,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167098 2024-12-09T06:19:27,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167098) roll requested 2024-12-09T06:19:27,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167105 2024-12-09T06:19:27,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167105 2024-12-09T06:19:27,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167105) roll requested 2024-12-09T06:19:27,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167114 2024-12-09T06:19:27,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167114 2024-12-09T06:19:27,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167114) roll requested 2024-12-09T06:19:27,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167120 2024-12-09T06:19:27,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167120 2024-12-09T06:19:27,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167120) roll requested 2024-12-09T06:19:27,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167127 2024-12-09T06:19:27,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167127 2024-12-09T06:19:27,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167127) roll requested 2024-12-09T06:19:27,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167135 2024-12-09T06:19:27,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167135 2024-12-09T06:19:27,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167135) roll requested 2024-12-09T06:19:27,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167144 2024-12-09T06:19:27,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167144 2024-12-09T06:19:27,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167144) roll requested 2024-12-09T06:19:27,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167153 2024-12-09T06:19:27,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167153 2024-12-09T06:19:27,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167153) roll requested 2024-12-09T06:19:27,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167162 2024-12-09T06:19:27,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167162 2024-12-09T06:19:27,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167162) roll requested 2024-12-09T06:19:27,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167173 2024-12-09T06:19:27,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167173 2024-12-09T06:19:27,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167173) roll requested 2024-12-09T06:19:27,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167181 2024-12-09T06:19:27,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167181 2024-12-09T06:19:27,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167181) roll requested 2024-12-09T06:19:27,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167187 2024-12-09T06:19:27,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167187 2024-12-09T06:19:27,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167187) roll requested 2024-12-09T06:19:27,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167198 2024-12-09T06:19:27,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167198 2024-12-09T06:19:27,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167198) roll requested 2024-12-09T06:19:27,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167206 2024-12-09T06:19:27,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167206 2024-12-09T06:19:27,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167206) roll requested 2024-12-09T06:19:27,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167219 2024-12-09T06:19:27,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167219 2024-12-09T06:19:27,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167219) roll requested 2024-12-09T06:19:27,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167229 2024-12-09T06:19:27,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167229 2024-12-09T06:19:27,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167229) roll requested 2024-12-09T06:19:27,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167235 2024-12-09T06:19:27,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167235 2024-12-09T06:19:27,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167235) roll requested 2024-12-09T06:19:27,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167241 2024-12-09T06:19:27,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167241 2024-12-09T06:19:27,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167241) roll requested 2024-12-09T06:19:27,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167246 2024-12-09T06:19:27,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167246 2024-12-09T06:19:27,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167246) roll requested 2024-12-09T06:19:27,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167253 2024-12-09T06:19:27,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167253 2024-12-09T06:19:27,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167253) roll requested 2024-12-09T06:19:27,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167258 2024-12-09T06:19:27,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167258 2024-12-09T06:19:27,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167258) roll requested 2024-12-09T06:19:27,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167263 2024-12-09T06:19:27,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167263 2024-12-09T06:19:27,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167263) roll requested 2024-12-09T06:19:27,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167347 2024-12-09T06:19:27,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167347 2024-12-09T06:19:27,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167347) roll requested 2024-12-09T06:19:27,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167359 2024-12-09T06:19:27,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167359 2024-12-09T06:19:27,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167359) roll requested 2024-12-09T06:19:27,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167365 2024-12-09T06:19:27,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167365 2024-12-09T06:19:27,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167365) roll requested 2024-12-09T06:19:27,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167371 2024-12-09T06:19:27,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167371 2024-12-09T06:19:27,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167371) roll requested 2024-12-09T06:19:27,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167380 2024-12-09T06:19:27,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167380 2024-12-09T06:19:27,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167380) roll requested 2024-12-09T06:19:27,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167385 2024-12-09T06:19:27,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167385 2024-12-09T06:19:27,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167385) roll requested 2024-12-09T06:19:27,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167391 2024-12-09T06:19:27,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167391 2024-12-09T06:19:27,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167391) roll requested 2024-12-09T06:19:27,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167397 2024-12-09T06:19:27,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167397 2024-12-09T06:19:27,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167397) roll requested 2024-12-09T06:19:27,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167404 2024-12-09T06:19:27,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167404 2024-12-09T06:19:27,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167404) roll requested 2024-12-09T06:19:27,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167412 2024-12-09T06:19:27,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167412 2024-12-09T06:19:27,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167412) roll requested 2024-12-09T06:19:27,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167421 2024-12-09T06:19:27,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167421 2024-12-09T06:19:27,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167421) roll requested 2024-12-09T06:19:27,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167430 2024-12-09T06:19:27,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167430 2024-12-09T06:19:27,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167430) roll requested 2024-12-09T06:19:27,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167439 2024-12-09T06:19:27,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167439 2024-12-09T06:19:27,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167439) roll requested 2024-12-09T06:19:27,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167448 2024-12-09T06:19:27,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167448 2024-12-09T06:19:27,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167448) roll requested 2024-12-09T06:19:27,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167456 2024-12-09T06:19:27,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167456 2024-12-09T06:19:27,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167456) roll requested 2024-12-09T06:19:27,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167464 2024-12-09T06:19:27,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167464 2024-12-09T06:19:27,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167464) roll requested 2024-12-09T06:19:27,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167471 2024-12-09T06:19:27,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167471 2024-12-09T06:19:27,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167471) roll requested 2024-12-09T06:19:27,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167480 2024-12-09T06:19:27,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167480 2024-12-09T06:19:27,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167480) roll requested 2024-12-09T06:19:27,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167489 2024-12-09T06:19:27,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167489 2024-12-09T06:19:27,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167489) roll requested 2024-12-09T06:19:27,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167497 2024-12-09T06:19:27,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167497 2024-12-09T06:19:27,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167497) roll requested 2024-12-09T06:19:27,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167503 2024-12-09T06:19:27,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167503 2024-12-09T06:19:27,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167503) roll requested 2024-12-09T06:19:27,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167512 2024-12-09T06:19:27,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167512 2024-12-09T06:19:27,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167512) roll requested 2024-12-09T06:19:27,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167520 2024-12-09T06:19:27,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167520 2024-12-09T06:19:27,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167520) roll requested 2024-12-09T06:19:27,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167529 2024-12-09T06:19:27,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167529 2024-12-09T06:19:27,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167529) roll requested 2024-12-09T06:19:27,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167537 2024-12-09T06:19:27,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167537 2024-12-09T06:19:27,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167537) roll requested 2024-12-09T06:19:27,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167544 2024-12-09T06:19:27,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167544 2024-12-09T06:19:27,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167544) roll requested 2024-12-09T06:19:27,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167553 2024-12-09T06:19:27,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167553 2024-12-09T06:19:27,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167553) roll requested 2024-12-09T06:19:27,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167562 2024-12-09T06:19:27,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167562 2024-12-09T06:19:27,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167562) roll requested 2024-12-09T06:19:27,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167573 2024-12-09T06:19:27,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167573 2024-12-09T06:19:27,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167573) roll requested 2024-12-09T06:19:27,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167581 2024-12-09T06:19:27,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167581 2024-12-09T06:19:27,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167581) roll requested 2024-12-09T06:19:27,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167590 2024-12-09T06:19:27,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167590 2024-12-09T06:19:27,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167590) roll requested 2024-12-09T06:19:27,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167599 2024-12-09T06:19:27,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167599 2024-12-09T06:19:27,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167599) roll requested 2024-12-09T06:19:27,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167609 2024-12-09T06:19:27,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167609 2024-12-09T06:19:27,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167609) roll requested 2024-12-09T06:19:27,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167619 2024-12-09T06:19:27,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167619 2024-12-09T06:19:27,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167619) roll requested 2024-12-09T06:19:27,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167628 2024-12-09T06:19:27,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167628 2024-12-09T06:19:27,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167628) roll requested 2024-12-09T06:19:27,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167634 2024-12-09T06:19:27,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167634 2024-12-09T06:19:27,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167634) roll requested 2024-12-09T06:19:27,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167641 2024-12-09T06:19:27,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167641 2024-12-09T06:19:27,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167641) roll requested 2024-12-09T06:19:27,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167647 2024-12-09T06:19:27,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167647 2024-12-09T06:19:27,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167647) roll requested 2024-12-09T06:19:27,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167653 2024-12-09T06:19:27,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167653 2024-12-09T06:19:27,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167653) roll requested 2024-12-09T06:19:27,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167659 2024-12-09T06:19:27,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167659 2024-12-09T06:19:27,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167659) roll requested 2024-12-09T06:19:27,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167667 2024-12-09T06:19:27,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167667 2024-12-09T06:19:27,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167667) roll requested 2024-12-09T06:19:27,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167676 2024-12-09T06:19:27,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167676 2024-12-09T06:19:27,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167676) roll requested 2024-12-09T06:19:27,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167686 2024-12-09T06:19:27,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167686 2024-12-09T06:19:27,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167686) roll requested 2024-12-09T06:19:27,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167696 2024-12-09T06:19:27,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167696 2024-12-09T06:19:27,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167696) roll requested 2024-12-09T06:19:27,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167706 2024-12-09T06:19:27,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167706 2024-12-09T06:19:27,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167706) roll requested 2024-12-09T06:19:27,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167716 2024-12-09T06:19:27,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167716 2024-12-09T06:19:27,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167716) roll requested 2024-12-09T06:19:27,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167724 2024-12-09T06:19:27,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167724 2024-12-09T06:19:27,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167724) roll requested 2024-12-09T06:19:27,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167732 2024-12-09T06:19:27,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167732 2024-12-09T06:19:27,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167732) roll requested 2024-12-09T06:19:27,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167740 2024-12-09T06:19:27,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167740 2024-12-09T06:19:27,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167740) roll requested 2024-12-09T06:19:27,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167748 2024-12-09T06:19:27,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167748 2024-12-09T06:19:27,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167748) roll requested 2024-12-09T06:19:27,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167757 2024-12-09T06:19:27,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167757 2024-12-09T06:19:27,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167757) roll requested 2024-12-09T06:19:27,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167767 2024-12-09T06:19:27,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167767 2024-12-09T06:19:27,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167767) roll requested 2024-12-09T06:19:27,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167776 2024-12-09T06:19:27,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167776 2024-12-09T06:19:27,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167776) roll requested 2024-12-09T06:19:27,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167784 2024-12-09T06:19:27,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167784 2024-12-09T06:19:27,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167784) roll requested 2024-12-09T06:19:27,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167791 2024-12-09T06:19:27,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167791 2024-12-09T06:19:27,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167791) roll requested 2024-12-09T06:19:27,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167797 2024-12-09T06:19:27,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167797 2024-12-09T06:19:27,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167797) roll requested 2024-12-09T06:19:27,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167806 2024-12-09T06:19:27,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167806 2024-12-09T06:19:27,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167806) roll requested 2024-12-09T06:19:27,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167814 2024-12-09T06:19:27,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167814 2024-12-09T06:19:27,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167814) roll requested 2024-12-09T06:19:27,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167822 2024-12-09T06:19:27,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167822 2024-12-09T06:19:27,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167822) roll requested 2024-12-09T06:19:27,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167830 2024-12-09T06:19:27,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167830 2024-12-09T06:19:27,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167830) roll requested 2024-12-09T06:19:27,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167841 2024-12-09T06:19:27,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167841 2024-12-09T06:19:27,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167841) roll requested 2024-12-09T06:19:27,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167851 2024-12-09T06:19:27,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167851 2024-12-09T06:19:27,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167851) roll requested 2024-12-09T06:19:27,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167860 2024-12-09T06:19:27,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167860 2024-12-09T06:19:27,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167860) roll requested 2024-12-09T06:19:27,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167871 2024-12-09T06:19:27,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167871 2024-12-09T06:19:27,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167871) roll requested 2024-12-09T06:19:27,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167881 2024-12-09T06:19:27,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167881 2024-12-09T06:19:27,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167881) roll requested 2024-12-09T06:19:27,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167888 2024-12-09T06:19:27,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167888 2024-12-09T06:19:27,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167888) roll requested 2024-12-09T06:19:27,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167895 2024-12-09T06:19:27,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167895 2024-12-09T06:19:27,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167895) roll requested 2024-12-09T06:19:27,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167903 2024-12-09T06:19:27,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167903 2024-12-09T06:19:27,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167903) roll requested 2024-12-09T06:19:27,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167910 2024-12-09T06:19:27,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167910 2024-12-09T06:19:27,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167910) roll requested 2024-12-09T06:19:27,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167917 2024-12-09T06:19:27,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167917 2024-12-09T06:19:27,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167917) roll requested 2024-12-09T06:19:27,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167926 2024-12-09T06:19:27,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167926 2024-12-09T06:19:27,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167926) roll requested 2024-12-09T06:19:27,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167936 2024-12-09T06:19:27,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167936 2024-12-09T06:19:27,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167936) roll requested 2024-12-09T06:19:27,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167946 2024-12-09T06:19:27,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167946 2024-12-09T06:19:27,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167946) roll requested 2024-12-09T06:19:27,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167954 2024-12-09T06:19:27,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167954 2024-12-09T06:19:27,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167954) roll requested 2024-12-09T06:19:27,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167963 2024-12-09T06:19:27,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167963 2024-12-09T06:19:27,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167963) roll requested 2024-12-09T06:19:27,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167973 2024-12-09T06:19:27,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167973 2024-12-09T06:19:27,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167973) roll requested 2024-12-09T06:19:27,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167982 2024-12-09T06:19:27,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167982 2024-12-09T06:19:27,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167982) roll requested 2024-12-09T06:19:27,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167991 2024-12-09T06:19:27,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:27,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167991 2024-12-09T06:19:27,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:27,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:27,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:27,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:27,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167991) roll requested 2024-12-09T06:19:27,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725167998 2024-12-09T06:19:28,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167998 2024-12-09T06:19:28,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725167998) roll requested 2024-12-09T06:19:28,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168006 2024-12-09T06:19:28,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725167998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168006 2024-12-09T06:19:28,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168006) roll requested 2024-12-09T06:19:28,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168015 2024-12-09T06:19:28,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168015 2024-12-09T06:19:28,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168015) roll requested 2024-12-09T06:19:28,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168023 2024-12-09T06:19:28,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168023 2024-12-09T06:19:28,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168023) roll requested 2024-12-09T06:19:28,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168031 2024-12-09T06:19:28,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168031 2024-12-09T06:19:28,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168031) roll requested 2024-12-09T06:19:28,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168040 2024-12-09T06:19:28,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168040 2024-12-09T06:19:28,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168040) roll requested 2024-12-09T06:19:28,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168048 2024-12-09T06:19:28,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168048 2024-12-09T06:19:28,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168048) roll requested 2024-12-09T06:19:28,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168057 2024-12-09T06:19:28,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168057 2024-12-09T06:19:28,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168057) roll requested 2024-12-09T06:19:28,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168065 2024-12-09T06:19:28,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168065 2024-12-09T06:19:28,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168065) roll requested 2024-12-09T06:19:28,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168074 2024-12-09T06:19:28,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168074 2024-12-09T06:19:28,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168074) roll requested 2024-12-09T06:19:28,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168081 2024-12-09T06:19:28,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168081 2024-12-09T06:19:28,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168081) roll requested 2024-12-09T06:19:28,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168088 2024-12-09T06:19:28,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168088 2024-12-09T06:19:28,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168088) roll requested 2024-12-09T06:19:28,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168096 2024-12-09T06:19:28,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168096 2024-12-09T06:19:28,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168096) roll requested 2024-12-09T06:19:28,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168106 2024-12-09T06:19:28,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168106 2024-12-09T06:19:28,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168106) roll requested 2024-12-09T06:19:28,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168115 2024-12-09T06:19:28,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168115 2024-12-09T06:19:28,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168115) roll requested 2024-12-09T06:19:28,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168122 2024-12-09T06:19:28,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168122 2024-12-09T06:19:28,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168122) roll requested 2024-12-09T06:19:28,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168130 2024-12-09T06:19:28,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168130 2024-12-09T06:19:28,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168130) roll requested 2024-12-09T06:19:28,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168138 2024-12-09T06:19:28,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168138 2024-12-09T06:19:28,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168138) roll requested 2024-12-09T06:19:28,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168146 2024-12-09T06:19:28,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168146 2024-12-09T06:19:28,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168146) roll requested 2024-12-09T06:19:28,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168156 2024-12-09T06:19:28,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168156 2024-12-09T06:19:28,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168156) roll requested 2024-12-09T06:19:28,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168166 2024-12-09T06:19:28,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168166 2024-12-09T06:19:28,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168166) roll requested 2024-12-09T06:19:28,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168176 2024-12-09T06:19:28,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168176 2024-12-09T06:19:28,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168176) roll requested 2024-12-09T06:19:28,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168183 2024-12-09T06:19:28,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168183 2024-12-09T06:19:28,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168183) roll requested 2024-12-09T06:19:28,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168191 2024-12-09T06:19:28,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168191 2024-12-09T06:19:28,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168191) roll requested 2024-12-09T06:19:28,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168200 2024-12-09T06:19:28,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168200 2024-12-09T06:19:28,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168200) roll requested 2024-12-09T06:19:28,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168207 2024-12-09T06:19:28,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168207 2024-12-09T06:19:28,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168207) roll requested 2024-12-09T06:19:28,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168212 2024-12-09T06:19:28,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168212 2024-12-09T06:19:28,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168212) roll requested 2024-12-09T06:19:28,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168219 2024-12-09T06:19:28,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168219 2024-12-09T06:19:28,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168219) roll requested 2024-12-09T06:19:28,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168227 2024-12-09T06:19:28,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168227 2024-12-09T06:19:28,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168227) roll requested 2024-12-09T06:19:28,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168235 2024-12-09T06:19:28,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168235 2024-12-09T06:19:28,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168235) roll requested 2024-12-09T06:19:28,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168244 2024-12-09T06:19:28,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168244 2024-12-09T06:19:28,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168244) roll requested 2024-12-09T06:19:28,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168253 2024-12-09T06:19:28,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168253 2024-12-09T06:19:28,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168253) roll requested 2024-12-09T06:19:28,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168264 2024-12-09T06:19:28,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168264 2024-12-09T06:19:28,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168264) roll requested 2024-12-09T06:19:28,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168274 2024-12-09T06:19:28,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168274 2024-12-09T06:19:28,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168274) roll requested 2024-12-09T06:19:28,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168281 2024-12-09T06:19:28,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168281 2024-12-09T06:19:28,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168281) roll requested 2024-12-09T06:19:28,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168288 2024-12-09T06:19:28,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168288 2024-12-09T06:19:28,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168288) roll requested 2024-12-09T06:19:28,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168296 2024-12-09T06:19:28,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168296 2024-12-09T06:19:28,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168296) roll requested 2024-12-09T06:19:28,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168304 2024-12-09T06:19:28,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168304 2024-12-09T06:19:28,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168304) roll requested 2024-12-09T06:19:28,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168314 2024-12-09T06:19:28,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168314 2024-12-09T06:19:28,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168314) roll requested 2024-12-09T06:19:28,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168322 2024-12-09T06:19:28,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168322 2024-12-09T06:19:28,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168322) roll requested 2024-12-09T06:19:28,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168331 2024-12-09T06:19:28,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168331 2024-12-09T06:19:28,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168331) roll requested 2024-12-09T06:19:28,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168341 2024-12-09T06:19:28,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168341 2024-12-09T06:19:28,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168341) roll requested 2024-12-09T06:19:28,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168352 2024-12-09T06:19:28,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168352 2024-12-09T06:19:28,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168352) roll requested 2024-12-09T06:19:28,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168364 2024-12-09T06:19:28,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168364 2024-12-09T06:19:28,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168364) roll requested 2024-12-09T06:19:28,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168371 2024-12-09T06:19:28,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168371 2024-12-09T06:19:28,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168371) roll requested 2024-12-09T06:19:28,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168380 2024-12-09T06:19:28,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168380 2024-12-09T06:19:28,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168380) roll requested 2024-12-09T06:19:28,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168389 2024-12-09T06:19:28,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168389 2024-12-09T06:19:28,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168389) roll requested 2024-12-09T06:19:28,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168398 2024-12-09T06:19:28,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168398 2024-12-09T06:19:28,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168398) roll requested 2024-12-09T06:19:28,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168407 2024-12-09T06:19:28,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168407 2024-12-09T06:19:28,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168407) roll requested 2024-12-09T06:19:28,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168415 2024-12-09T06:19:28,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168415 2024-12-09T06:19:28,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168415) roll requested 2024-12-09T06:19:28,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168425 2024-12-09T06:19:28,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168425 2024-12-09T06:19:28,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168425) roll requested 2024-12-09T06:19:28,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168432 2024-12-09T06:19:28,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168432 2024-12-09T06:19:28,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168432) roll requested 2024-12-09T06:19:28,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168439 2024-12-09T06:19:28,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168439 2024-12-09T06:19:28,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168439) roll requested 2024-12-09T06:19:28,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168446 2024-12-09T06:19:28,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168446 2024-12-09T06:19:28,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168446) roll requested 2024-12-09T06:19:28,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168454 2024-12-09T06:19:28,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168454 2024-12-09T06:19:28,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168454) roll requested 2024-12-09T06:19:28,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168462 2024-12-09T06:19:28,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168462 2024-12-09T06:19:28,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168462) roll requested 2024-12-09T06:19:28,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168470 2024-12-09T06:19:28,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168470 2024-12-09T06:19:28,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168470) roll requested 2024-12-09T06:19:28,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168478 2024-12-09T06:19:28,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168478 2024-12-09T06:19:28,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168478) roll requested 2024-12-09T06:19:28,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168487 2024-12-09T06:19:28,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168487 2024-12-09T06:19:28,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168487) roll requested 2024-12-09T06:19:28,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168493 2024-12-09T06:19:28,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168493 2024-12-09T06:19:28,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168493) roll requested 2024-12-09T06:19:28,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168498 2024-12-09T06:19:28,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168498 2024-12-09T06:19:28,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168498) roll requested 2024-12-09T06:19:28,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168505 2024-12-09T06:19:28,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168505 2024-12-09T06:19:28,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168505) roll requested 2024-12-09T06:19:28,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168512 2024-12-09T06:19:28,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168512 2024-12-09T06:19:28,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168512) roll requested 2024-12-09T06:19:28,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168520 2024-12-09T06:19:28,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168520 2024-12-09T06:19:28,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168520) roll requested 2024-12-09T06:19:28,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168529 2024-12-09T06:19:28,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168529 2024-12-09T06:19:28,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168529) roll requested 2024-12-09T06:19:28,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168537 2024-12-09T06:19:28,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168537 2024-12-09T06:19:28,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168537) roll requested 2024-12-09T06:19:28,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168547 2024-12-09T06:19:28,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168547 2024-12-09T06:19:28,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168547) roll requested 2024-12-09T06:19:28,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168556 2024-12-09T06:19:28,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168556 2024-12-09T06:19:28,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168556) roll requested 2024-12-09T06:19:28,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168566 2024-12-09T06:19:28,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168566 2024-12-09T06:19:28,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168566) roll requested 2024-12-09T06:19:28,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168576 2024-12-09T06:19:28,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168576 2024-12-09T06:19:28,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168576) roll requested 2024-12-09T06:19:28,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168585 2024-12-09T06:19:28,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168585 2024-12-09T06:19:28,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168585) roll requested 2024-12-09T06:19:28,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168594 2024-12-09T06:19:28,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168594 2024-12-09T06:19:28,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168594) roll requested 2024-12-09T06:19:28,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168604 2024-12-09T06:19:28,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168604 2024-12-09T06:19:28,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168604) roll requested 2024-12-09T06:19:28,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168613 2024-12-09T06:19:28,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168613 2024-12-09T06:19:28,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168613) roll requested 2024-12-09T06:19:28,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168622 2024-12-09T06:19:28,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168622 2024-12-09T06:19:28,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168622) roll requested 2024-12-09T06:19:28,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168631 2024-12-09T06:19:28,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168631 2024-12-09T06:19:28,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168631) roll requested 2024-12-09T06:19:28,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168641 2024-12-09T06:19:28,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168641 2024-12-09T06:19:28,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168641) roll requested 2024-12-09T06:19:28,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168648 2024-12-09T06:19:28,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168648 2024-12-09T06:19:28,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168648) roll requested 2024-12-09T06:19:28,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168657 2024-12-09T06:19:28,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168657 2024-12-09T06:19:28,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168657) roll requested 2024-12-09T06:19:28,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168670 2024-12-09T06:19:28,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168670 2024-12-09T06:19:28,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168670) roll requested 2024-12-09T06:19:28,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168679 2024-12-09T06:19:28,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168679 2024-12-09T06:19:28,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168679) roll requested 2024-12-09T06:19:28,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168689 2024-12-09T06:19:28,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168689 2024-12-09T06:19:28,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168689) roll requested 2024-12-09T06:19:28,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168696 2024-12-09T06:19:28,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168696 2024-12-09T06:19:28,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168696) roll requested 2024-12-09T06:19:28,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168701 2024-12-09T06:19:28,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168701 2024-12-09T06:19:28,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168701) roll requested 2024-12-09T06:19:28,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168707 2024-12-09T06:19:28,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168707 2024-12-09T06:19:28,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168707) roll requested 2024-12-09T06:19:28,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168712 2024-12-09T06:19:28,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168712 2024-12-09T06:19:28,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168712) roll requested 2024-12-09T06:19:28,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168718 2024-12-09T06:19:28,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168718 2024-12-09T06:19:28,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168718) roll requested 2024-12-09T06:19:28,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168725 2024-12-09T06:19:28,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168725 2024-12-09T06:19:28,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168725) roll requested 2024-12-09T06:19:28,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168730 2024-12-09T06:19:28,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168730 2024-12-09T06:19:28,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168730) roll requested 2024-12-09T06:19:28,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168736 2024-12-09T06:19:28,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168736 2024-12-09T06:19:28,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168736) roll requested 2024-12-09T06:19:28,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168741 2024-12-09T06:19:28,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168741 2024-12-09T06:19:28,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168741) roll requested 2024-12-09T06:19:28,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168747 2024-12-09T06:19:28,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168747 2024-12-09T06:19:28,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168747) roll requested 2024-12-09T06:19:28,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168753 2024-12-09T06:19:28,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168753 2024-12-09T06:19:28,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168753) roll requested 2024-12-09T06:19:28,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168758 2024-12-09T06:19:28,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168758 2024-12-09T06:19:28,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168758) roll requested 2024-12-09T06:19:28,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168764 2024-12-09T06:19:28,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168764 2024-12-09T06:19:28,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168764) roll requested 2024-12-09T06:19:28,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168769 2024-12-09T06:19:28,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168769 2024-12-09T06:19:28,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168769) roll requested 2024-12-09T06:19:28,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168774 2024-12-09T06:19:28,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168774 2024-12-09T06:19:28,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168774) roll requested 2024-12-09T06:19:28,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168781 2024-12-09T06:19:28,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168781 2024-12-09T06:19:28,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168781) roll requested 2024-12-09T06:19:28,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168787 2024-12-09T06:19:28,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168787 2024-12-09T06:19:28,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168787) roll requested 2024-12-09T06:19:28,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168796 2024-12-09T06:19:28,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168796 2024-12-09T06:19:28,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168796) roll requested 2024-12-09T06:19:28,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168804 2024-12-09T06:19:28,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168804 2024-12-09T06:19:28,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168804) roll requested 2024-12-09T06:19:28,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168811 2024-12-09T06:19:28,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168811 2024-12-09T06:19:28,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168811) roll requested 2024-12-09T06:19:28,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168820 2024-12-09T06:19:28,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168820 2024-12-09T06:19:28,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168820) roll requested 2024-12-09T06:19:28,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168828 2024-12-09T06:19:28,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168828 2024-12-09T06:19:28,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168828) roll requested 2024-12-09T06:19:28,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168835 2024-12-09T06:19:28,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168835 2024-12-09T06:19:28,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168835) roll requested 2024-12-09T06:19:28,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168844 2024-12-09T06:19:28,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168844 2024-12-09T06:19:28,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168844) roll requested 2024-12-09T06:19:28,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168855 2024-12-09T06:19:28,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168855 2024-12-09T06:19:28,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168855) roll requested 2024-12-09T06:19:28,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168864 2024-12-09T06:19:28,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168864 2024-12-09T06:19:28,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168864) roll requested 2024-12-09T06:19:28,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168872 2024-12-09T06:19:28,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168872 2024-12-09T06:19:28,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168872) roll requested 2024-12-09T06:19:28,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168880 2024-12-09T06:19:28,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168880 2024-12-09T06:19:28,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168880) roll requested 2024-12-09T06:19:28,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168888 2024-12-09T06:19:28,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168888 2024-12-09T06:19:28,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168888) roll requested 2024-12-09T06:19:28,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168897 2024-12-09T06:19:28,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168897 2024-12-09T06:19:28,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168897) roll requested 2024-12-09T06:19:28,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168906 2024-12-09T06:19:28,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168906 2024-12-09T06:19:28,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168906) roll requested 2024-12-09T06:19:28,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168913 2024-12-09T06:19:28,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168913 2024-12-09T06:19:28,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168913) roll requested 2024-12-09T06:19:28,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168921 2024-12-09T06:19:28,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168921 2024-12-09T06:19:28,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168921) roll requested 2024-12-09T06:19:28,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168929 2024-12-09T06:19:28,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168929 2024-12-09T06:19:28,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168929) roll requested 2024-12-09T06:19:28,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168939 2024-12-09T06:19:28,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168939 2024-12-09T06:19:28,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168939) roll requested 2024-12-09T06:19:28,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168949 2024-12-09T06:19:28,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168949 2024-12-09T06:19:28,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168949) roll requested 2024-12-09T06:19:28,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168958 2024-12-09T06:19:28,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168958 2024-12-09T06:19:28,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168958) roll requested 2024-12-09T06:19:28,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168965 2024-12-09T06:19:28,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168965 2024-12-09T06:19:28,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168965) roll requested 2024-12-09T06:19:28,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168972 2024-12-09T06:19:28,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168972 2024-12-09T06:19:28,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168972) roll requested 2024-12-09T06:19:28,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168979 2024-12-09T06:19:28,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168979 2024-12-09T06:19:28,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168979) roll requested 2024-12-09T06:19:28,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168987 2024-12-09T06:19:28,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:28,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168987 2024-12-09T06:19:28,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:28,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:28,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:28,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:28,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168987) roll requested 2024-12-09T06:19:28,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725168995 2024-12-09T06:19:29,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168995 2024-12-09T06:19:29,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725168995) roll requested 2024-12-09T06:19:29,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169005 2024-12-09T06:19:29,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725168995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169005 2024-12-09T06:19:29,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169005) roll requested 2024-12-09T06:19:29,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169014 2024-12-09T06:19:29,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169014 2024-12-09T06:19:29,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169014) roll requested 2024-12-09T06:19:29,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169025 2024-12-09T06:19:29,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169025 2024-12-09T06:19:29,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169025) roll requested 2024-12-09T06:19:29,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169034 2024-12-09T06:19:29,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169034 2024-12-09T06:19:29,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169034) roll requested 2024-12-09T06:19:29,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169042 2024-12-09T06:19:29,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169042 2024-12-09T06:19:29,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169042) roll requested 2024-12-09T06:19:29,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169053 2024-12-09T06:19:29,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169053 2024-12-09T06:19:29,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169053) roll requested 2024-12-09T06:19:29,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169059 2024-12-09T06:19:29,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169059 2024-12-09T06:19:29,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169059) roll requested 2024-12-09T06:19:29,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169065 2024-12-09T06:19:29,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169065 2024-12-09T06:19:29,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169065) roll requested 2024-12-09T06:19:29,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169072 2024-12-09T06:19:29,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169072 2024-12-09T06:19:29,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169072) roll requested 2024-12-09T06:19:29,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169079 2024-12-09T06:19:29,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169079 2024-12-09T06:19:29,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169079) roll requested 2024-12-09T06:19:29,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169087 2024-12-09T06:19:29,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169087 2024-12-09T06:19:29,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169087) roll requested 2024-12-09T06:19:29,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169096 2024-12-09T06:19:29,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169096 2024-12-09T06:19:29,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169096) roll requested 2024-12-09T06:19:29,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169105 2024-12-09T06:19:29,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169105 2024-12-09T06:19:29,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169105) roll requested 2024-12-09T06:19:29,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169114 2024-12-09T06:19:29,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169114 2024-12-09T06:19:29,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169114) roll requested 2024-12-09T06:19:29,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169119 2024-12-09T06:19:29,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169119 2024-12-09T06:19:29,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169119) roll requested 2024-12-09T06:19:29,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169125 2024-12-09T06:19:29,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169125 2024-12-09T06:19:29,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169125) roll requested 2024-12-09T06:19:29,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169131 2024-12-09T06:19:29,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169131 2024-12-09T06:19:29,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169131) roll requested 2024-12-09T06:19:29,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169137 2024-12-09T06:19:29,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169137 2024-12-09T06:19:29,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169137) roll requested 2024-12-09T06:19:29,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169143 2024-12-09T06:19:29,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169143 2024-12-09T06:19:29,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169143) roll requested 2024-12-09T06:19:29,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169148 2024-12-09T06:19:29,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169148 2024-12-09T06:19:29,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169148) roll requested 2024-12-09T06:19:29,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169153 2024-12-09T06:19:29,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169153 2024-12-09T06:19:29,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169153) roll requested 2024-12-09T06:19:29,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169159 2024-12-09T06:19:29,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169159 2024-12-09T06:19:29,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169159) roll requested 2024-12-09T06:19:29,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169166 2024-12-09T06:19:29,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169166 2024-12-09T06:19:29,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169166) roll requested 2024-12-09T06:19:29,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169173 2024-12-09T06:19:29,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169173 2024-12-09T06:19:29,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169173) roll requested 2024-12-09T06:19:29,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169180 2024-12-09T06:19:29,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169180 2024-12-09T06:19:29,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169180) roll requested 2024-12-09T06:19:29,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169188 2024-12-09T06:19:29,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169188 2024-12-09T06:19:29,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169188) roll requested 2024-12-09T06:19:29,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169197 2024-12-09T06:19:29,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169197 2024-12-09T06:19:29,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169197) roll requested 2024-12-09T06:19:29,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169205 2024-12-09T06:19:29,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169205 2024-12-09T06:19:29,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169205) roll requested 2024-12-09T06:19:29,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169210 2024-12-09T06:19:29,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169210 2024-12-09T06:19:29,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169210) roll requested 2024-12-09T06:19:29,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169215 2024-12-09T06:19:29,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169215 2024-12-09T06:19:29,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169215) roll requested 2024-12-09T06:19:29,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169223 2024-12-09T06:19:29,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169223 2024-12-09T06:19:29,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169223) roll requested 2024-12-09T06:19:29,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169230 2024-12-09T06:19:29,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169230 2024-12-09T06:19:29,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169230) roll requested 2024-12-09T06:19:29,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169238 2024-12-09T06:19:29,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169238 2024-12-09T06:19:29,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169238) roll requested 2024-12-09T06:19:29,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169246 2024-12-09T06:19:29,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169246 2024-12-09T06:19:29,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169246) roll requested 2024-12-09T06:19:29,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169256 2024-12-09T06:19:29,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169256 2024-12-09T06:19:29,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169256) roll requested 2024-12-09T06:19:29,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169266 2024-12-09T06:19:29,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169266 2024-12-09T06:19:29,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169266) roll requested 2024-12-09T06:19:29,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169273 2024-12-09T06:19:29,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169273 2024-12-09T06:19:29,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169273) roll requested 2024-12-09T06:19:29,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169279 2024-12-09T06:19:29,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169279 2024-12-09T06:19:29,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169279) roll requested 2024-12-09T06:19:29,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169286 2024-12-09T06:19:29,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169286 2024-12-09T06:19:29,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169286) roll requested 2024-12-09T06:19:29,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169294 2024-12-09T06:19:29,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169294 2024-12-09T06:19:29,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169294) roll requested 2024-12-09T06:19:29,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169302 2024-12-09T06:19:29,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169302 2024-12-09T06:19:29,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169302) roll requested 2024-12-09T06:19:29,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169310 2024-12-09T06:19:29,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169310 2024-12-09T06:19:29,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169310) roll requested 2024-12-09T06:19:29,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169318 2024-12-09T06:19:29,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169318 2024-12-09T06:19:29,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169318) roll requested 2024-12-09T06:19:29,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169326 2024-12-09T06:19:29,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169326 2024-12-09T06:19:29,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169326) roll requested 2024-12-09T06:19:29,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169336 2024-12-09T06:19:29,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169336 2024-12-09T06:19:29,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169336) roll requested 2024-12-09T06:19:29,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169346 2024-12-09T06:19:29,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169346 2024-12-09T06:19:29,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169346) roll requested 2024-12-09T06:19:29,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169355 2024-12-09T06:19:29,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169355 2024-12-09T06:19:29,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169355) roll requested 2024-12-09T06:19:29,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169366 2024-12-09T06:19:29,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169366 2024-12-09T06:19:29,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169366) roll requested 2024-12-09T06:19:29,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169375 2024-12-09T06:19:29,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169375 2024-12-09T06:19:29,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169375) roll requested 2024-12-09T06:19:29,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169382 2024-12-09T06:19:29,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169382 2024-12-09T06:19:29,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169382) roll requested 2024-12-09T06:19:29,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169392 2024-12-09T06:19:29,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169392 2024-12-09T06:19:29,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169392) roll requested 2024-12-09T06:19:29,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169399 2024-12-09T06:19:29,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169399 2024-12-09T06:19:29,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169399) roll requested 2024-12-09T06:19:29,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169407 2024-12-09T06:19:29,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169407 2024-12-09T06:19:29,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169407) roll requested 2024-12-09T06:19:29,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169417 2024-12-09T06:19:29,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169417 2024-12-09T06:19:29,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169417) roll requested 2024-12-09T06:19:29,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169426 2024-12-09T06:19:29,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169426 2024-12-09T06:19:29,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169426) roll requested 2024-12-09T06:19:29,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169435 2024-12-09T06:19:29,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169435 2024-12-09T06:19:29,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169435) roll requested 2024-12-09T06:19:29,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169444 2024-12-09T06:19:29,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169444 2024-12-09T06:19:29,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169444) roll requested 2024-12-09T06:19:29,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169452 2024-12-09T06:19:29,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169452 2024-12-09T06:19:29,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169452) roll requested 2024-12-09T06:19:29,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169462 2024-12-09T06:19:29,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169462 2024-12-09T06:19:29,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169462) roll requested 2024-12-09T06:19:29,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169470 2024-12-09T06:19:29,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169470 2024-12-09T06:19:29,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169470) roll requested 2024-12-09T06:19:29,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169477 2024-12-09T06:19:29,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169477 2024-12-09T06:19:29,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169477) roll requested 2024-12-09T06:19:29,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169485 2024-12-09T06:19:29,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169485 2024-12-09T06:19:29,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169485) roll requested 2024-12-09T06:19:29,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169492 2024-12-09T06:19:29,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169492 2024-12-09T06:19:29,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169492) roll requested 2024-12-09T06:19:29,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169500 2024-12-09T06:19:29,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169500 2024-12-09T06:19:29,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169500) roll requested 2024-12-09T06:19:29,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169509 2024-12-09T06:19:29,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169509 2024-12-09T06:19:29,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169509) roll requested 2024-12-09T06:19:29,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169517 2024-12-09T06:19:29,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169517 2024-12-09T06:19:29,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169517) roll requested 2024-12-09T06:19:29,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169525 2024-12-09T06:19:29,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169525 2024-12-09T06:19:29,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169525) roll requested 2024-12-09T06:19:29,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169533 2024-12-09T06:19:29,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169533 2024-12-09T06:19:29,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169533) roll requested 2024-12-09T06:19:29,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169544 2024-12-09T06:19:29,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169544 2024-12-09T06:19:29,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169544) roll requested 2024-12-09T06:19:29,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169553 2024-12-09T06:19:29,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169553 2024-12-09T06:19:29,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169553) roll requested 2024-12-09T06:19:29,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169561 2024-12-09T06:19:29,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169561 2024-12-09T06:19:29,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169561) roll requested 2024-12-09T06:19:29,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169567 2024-12-09T06:19:29,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169567 2024-12-09T06:19:29,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169567) roll requested 2024-12-09T06:19:29,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169573 2024-12-09T06:19:29,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169573 2024-12-09T06:19:29,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169573) roll requested 2024-12-09T06:19:29,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169580 2024-12-09T06:19:29,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169580 2024-12-09T06:19:29,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169580) roll requested 2024-12-09T06:19:29,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169588 2024-12-09T06:19:29,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169588 2024-12-09T06:19:29,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169588) roll requested 2024-12-09T06:19:29,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169596 2024-12-09T06:19:29,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169596 2024-12-09T06:19:29,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169596) roll requested 2024-12-09T06:19:29,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169606 2024-12-09T06:19:29,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169606 2024-12-09T06:19:29,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169606) roll requested 2024-12-09T06:19:29,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169616 2024-12-09T06:19:29,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169616 2024-12-09T06:19:29,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169616) roll requested 2024-12-09T06:19:29,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169625 2024-12-09T06:19:29,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169625 2024-12-09T06:19:29,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169625) roll requested 2024-12-09T06:19:29,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169634 2024-12-09T06:19:29,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169634 2024-12-09T06:19:29,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169634) roll requested 2024-12-09T06:19:29,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169640 2024-12-09T06:19:29,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169640 2024-12-09T06:19:29,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169640) roll requested 2024-12-09T06:19:29,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169645 2024-12-09T06:19:29,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169645 2024-12-09T06:19:29,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169645) roll requested 2024-12-09T06:19:29,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169650 2024-12-09T06:19:29,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169650 2024-12-09T06:19:29,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169650) roll requested 2024-12-09T06:19:29,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169657 2024-12-09T06:19:29,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169657 2024-12-09T06:19:29,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169657) roll requested 2024-12-09T06:19:29,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169665 2024-12-09T06:19:29,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169665 2024-12-09T06:19:29,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169665) roll requested 2024-12-09T06:19:29,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169673 2024-12-09T06:19:29,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169673 2024-12-09T06:19:29,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169673) roll requested 2024-12-09T06:19:29,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169679 2024-12-09T06:19:29,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169679 2024-12-09T06:19:29,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169679) roll requested 2024-12-09T06:19:29,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169686 2024-12-09T06:19:29,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169686 2024-12-09T06:19:29,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169686) roll requested 2024-12-09T06:19:29,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169694 2024-12-09T06:19:29,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169694 2024-12-09T06:19:29,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169694) roll requested 2024-12-09T06:19:29,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169703 2024-12-09T06:19:29,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169703 2024-12-09T06:19:29,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169703) roll requested 2024-12-09T06:19:29,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169714 2024-12-09T06:19:29,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169714 2024-12-09T06:19:29,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169714) roll requested 2024-12-09T06:19:29,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169724 2024-12-09T06:19:29,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169724 2024-12-09T06:19:29,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169724) roll requested 2024-12-09T06:19:29,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169734 2024-12-09T06:19:29,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169734 2024-12-09T06:19:29,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169734) roll requested 2024-12-09T06:19:29,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169740 2024-12-09T06:19:29,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169740 2024-12-09T06:19:29,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169740) roll requested 2024-12-09T06:19:29,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169747 2024-12-09T06:19:29,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169747 2024-12-09T06:19:29,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169747) roll requested 2024-12-09T06:19:29,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169753 2024-12-09T06:19:29,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169753 2024-12-09T06:19:29,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169753) roll requested 2024-12-09T06:19:29,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169760 2024-12-09T06:19:29,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169760 2024-12-09T06:19:29,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169760) roll requested 2024-12-09T06:19:29,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169769 2024-12-09T06:19:29,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169769 2024-12-09T06:19:29,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169769) roll requested 2024-12-09T06:19:29,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169776 2024-12-09T06:19:29,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169776 2024-12-09T06:19:29,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169776) roll requested 2024-12-09T06:19:29,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169785 2024-12-09T06:19:29,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169785 2024-12-09T06:19:29,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169785) roll requested 2024-12-09T06:19:29,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169794 2024-12-09T06:19:29,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169794 2024-12-09T06:19:29,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169794) roll requested 2024-12-09T06:19:29,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169802 2024-12-09T06:19:29,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169802 2024-12-09T06:19:29,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169802) roll requested 2024-12-09T06:19:29,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169811 2024-12-09T06:19:29,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169811 2024-12-09T06:19:29,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169811) roll requested 2024-12-09T06:19:29,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169820 2024-12-09T06:19:29,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169820 2024-12-09T06:19:29,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169820) roll requested 2024-12-09T06:19:29,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169828 2024-12-09T06:19:29,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169828 2024-12-09T06:19:29,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169828) roll requested 2024-12-09T06:19:29,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169838 2024-12-09T06:19:29,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169838 2024-12-09T06:19:29,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169838) roll requested 2024-12-09T06:19:29,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169846 2024-12-09T06:19:29,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169846 2024-12-09T06:19:29,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169846) roll requested 2024-12-09T06:19:29,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169855 2024-12-09T06:19:29,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169855 2024-12-09T06:19:29,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169855) roll requested 2024-12-09T06:19:29,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169865 2024-12-09T06:19:29,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169865 2024-12-09T06:19:29,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169865) roll requested 2024-12-09T06:19:29,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169875 2024-12-09T06:19:29,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169875 2024-12-09T06:19:29,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169875) roll requested 2024-12-09T06:19:29,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169884 2024-12-09T06:19:29,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169884 2024-12-09T06:19:29,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169884) roll requested 2024-12-09T06:19:29,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169893 2024-12-09T06:19:29,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169893 2024-12-09T06:19:29,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169893) roll requested 2024-12-09T06:19:29,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169902 2024-12-09T06:19:29,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169902 2024-12-09T06:19:29,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169902) roll requested 2024-12-09T06:19:29,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169910 2024-12-09T06:19:29,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169910 2024-12-09T06:19:29,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169910) roll requested 2024-12-09T06:19:29,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169918 2024-12-09T06:19:29,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169918 2024-12-09T06:19:29,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169918) roll requested 2024-12-09T06:19:29,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169926 2024-12-09T06:19:29,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169926 2024-12-09T06:19:29,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169926) roll requested 2024-12-09T06:19:29,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169934 2024-12-09T06:19:29,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169934 2024-12-09T06:19:29,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169934) roll requested 2024-12-09T06:19:29,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169941 2024-12-09T06:19:29,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169941 2024-12-09T06:19:29,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169941) roll requested 2024-12-09T06:19:29,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169950 2024-12-09T06:19:29,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169950 2024-12-09T06:19:29,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169950) roll requested 2024-12-09T06:19:29,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169958 2024-12-09T06:19:29,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169958 2024-12-09T06:19:29,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169958) roll requested 2024-12-09T06:19:29,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169966 2024-12-09T06:19:29,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169966 2024-12-09T06:19:29,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169966) roll requested 2024-12-09T06:19:29,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169975 2024-12-09T06:19:29,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169975 2024-12-09T06:19:29,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169975) roll requested 2024-12-09T06:19:29,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169983 2024-12-09T06:19:29,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169983 2024-12-09T06:19:29,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169983) roll requested 2024-12-09T06:19:29,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169989 2024-12-09T06:19:29,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:29,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169989 2024-12-09T06:19:29,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:29,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:29,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:29,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:29,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169989) roll requested 2024-12-09T06:19:29,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725169997 2024-12-09T06:19:30,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169997 2024-12-09T06:19:30,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725169997) roll requested 2024-12-09T06:19:30,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170006 2024-12-09T06:19:30,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725169997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170006 2024-12-09T06:19:30,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170006) roll requested 2024-12-09T06:19:30,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170018 2024-12-09T06:19:30,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170018 2024-12-09T06:19:30,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170018) roll requested 2024-12-09T06:19:30,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170027 2024-12-09T06:19:30,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170027 2024-12-09T06:19:30,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170027) roll requested 2024-12-09T06:19:30,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170037 2024-12-09T06:19:30,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170037 2024-12-09T06:19:30,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170037) roll requested 2024-12-09T06:19:30,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170046 2024-12-09T06:19:30,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170046 2024-12-09T06:19:30,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170046) roll requested 2024-12-09T06:19:30,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170053 2024-12-09T06:19:30,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170053 2024-12-09T06:19:30,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170053) roll requested 2024-12-09T06:19:30,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170061 2024-12-09T06:19:30,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170061 2024-12-09T06:19:30,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170061) roll requested 2024-12-09T06:19:30,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170069 2024-12-09T06:19:30,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170069 2024-12-09T06:19:30,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170069) roll requested 2024-12-09T06:19:30,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170078 2024-12-09T06:19:30,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170078 2024-12-09T06:19:30,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170078) roll requested 2024-12-09T06:19:30,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170084 2024-12-09T06:19:30,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170084 2024-12-09T06:19:30,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170084) roll requested 2024-12-09T06:19:30,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170093 2024-12-09T06:19:30,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170093 2024-12-09T06:19:30,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170093) roll requested 2024-12-09T06:19:30,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170101 2024-12-09T06:19:30,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170101 2024-12-09T06:19:30,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170101) roll requested 2024-12-09T06:19:30,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170107 2024-12-09T06:19:30,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170107 2024-12-09T06:19:30,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170107) roll requested 2024-12-09T06:19:30,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170113 2024-12-09T06:19:30,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170113 2024-12-09T06:19:30,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170113) roll requested 2024-12-09T06:19:30,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170121 2024-12-09T06:19:30,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170121 2024-12-09T06:19:30,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170121) roll requested 2024-12-09T06:19:30,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170129 2024-12-09T06:19:30,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170129 2024-12-09T06:19:30,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170129) roll requested 2024-12-09T06:19:30,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170137 2024-12-09T06:19:30,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170137 2024-12-09T06:19:30,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170137) roll requested 2024-12-09T06:19:30,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170146 2024-12-09T06:19:30,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170146 2024-12-09T06:19:30,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170146) roll requested 2024-12-09T06:19:30,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170154 2024-12-09T06:19:30,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170154 2024-12-09T06:19:30,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170154) roll requested 2024-12-09T06:19:30,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170163 2024-12-09T06:19:30,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170163 2024-12-09T06:19:30,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170163) roll requested 2024-12-09T06:19:30,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170171 2024-12-09T06:19:30,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170171 2024-12-09T06:19:30,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170171) roll requested 2024-12-09T06:19:30,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170179 2024-12-09T06:19:30,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170179 2024-12-09T06:19:30,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170179) roll requested 2024-12-09T06:19:30,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170189 2024-12-09T06:19:30,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170189 2024-12-09T06:19:30,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170189) roll requested 2024-12-09T06:19:30,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170200 2024-12-09T06:19:30,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170200 2024-12-09T06:19:30,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170200) roll requested 2024-12-09T06:19:30,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170210 2024-12-09T06:19:30,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170210 2024-12-09T06:19:30,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170210) roll requested 2024-12-09T06:19:30,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170221 2024-12-09T06:19:30,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170221 2024-12-09T06:19:30,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170221) roll requested 2024-12-09T06:19:30,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170229 2024-12-09T06:19:30,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170229 2024-12-09T06:19:30,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170229) roll requested 2024-12-09T06:19:30,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170237 2024-12-09T06:19:30,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170237 2024-12-09T06:19:30,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1283, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170237) roll requested 2024-12-09T06:19:30,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170245 2024-12-09T06:19:30,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170245 2024-12-09T06:19:30,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1284, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170245) roll requested 2024-12-09T06:19:30,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170253 2024-12-09T06:19:30,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170253 2024-12-09T06:19:30,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1285, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170253) roll requested 2024-12-09T06:19:30,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170261 2024-12-09T06:19:30,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170261 2024-12-09T06:19:30,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1286, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170261) roll requested 2024-12-09T06:19:30,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170270 2024-12-09T06:19:30,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170270 2024-12-09T06:19:30,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1287, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170270) roll requested 2024-12-09T06:19:30,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170278 2024-12-09T06:19:30,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170278 2024-12-09T06:19:30,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1288, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170278) roll requested 2024-12-09T06:19:30,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170286 2024-12-09T06:19:30,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170286 2024-12-09T06:19:30,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1289, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170286) roll requested 2024-12-09T06:19:30,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170295 2024-12-09T06:19:30,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170295 2024-12-09T06:19:30,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1290, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170295) roll requested 2024-12-09T06:19:30,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170303 2024-12-09T06:19:30,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170303 2024-12-09T06:19:30,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1291, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170303) roll requested 2024-12-09T06:19:30,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170313 2024-12-09T06:19:30,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170313 2024-12-09T06:19:30,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T06:19:30,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1292, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733725170313) roll requested 2024-12-09T06:19:30,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733725170321 2024-12-09T06:19:30,324 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:19:30,326 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aea0f23f7b5f4c66013ed1db76b82ad1, disabling compactions & flushes 2024-12-09T06:19:30,327 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1. 2024-12-09T06:19:30,327 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1. 2024-12-09T06:19:30,327 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1. after waiting 0 ms 2024-12-09T06:19:30,327 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1. 2024-12-09T06:19:30,334 WARN [LogRoller {}] util.Shell(1106): Interrupted while joining on: Thread[Thread-7854,5,] java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.util.Shell.joinThread(Shell.java:1103) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1063) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] 2024-12-09T06:19:30,335 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1. 2024-12-09T06:19:30,335 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1733725139987.aea0f23f7b5f4c66013ed1db76b82ad1. 2024-12-09T06:19:30,335 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aea0f23f7b5f4c66013ed1db76b82ad1: Waiting for close lock at 1733725170326Disabling compacts and flushes for region at 1733725170326Disabling writes for close at 1733725170327 (+1 ms)Writing region close event to WAL at 1733725170334 (+7 ms)Closed at 1733725170335 (+1 ms) 2024-12-09T06:19:30,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733725170321 2024-12-09T06:19:30,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T06:19:30,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1293, max=55; forcing (partial) flush of 1 region(s): aea0f23f7b5f4c66013ed1db76b82ad1[MyCF] 2024-12-09T06:19:30,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of aea0f23f7b5f4c66013ed1db76b82ad1, because it is not online on us 2024-12-09T06:19:30,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,342 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T06:19:30,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T06:19:30,646 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1733725170321) 2024-12-09T06:19:30,647 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04f238f0-c2cf-3436-91ed-1190a6a36185 2024-12-09T06:19:31,194 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=428 (was 543), ProcessCount=11 (was 11), AvailableMemoryMB=5649 (was 6499)